From 3c6a6b4d2fca9bb273ec401cc75ec60cba0f5db3 Mon Sep 17 00:00:00 2001
From: ehila <ehila@redhat.com>
Date: Fri, 23 Jun 2023 13:37:24 -0400
Subject: [PATCH 1/8] feat: initial implementation of storage migration

Signed-off-by: ehila <ehila@redhat.com>
---
 pkg/admin/migration/migration.go | 282 +++++++++++++++++++++++++++++++
 pkg/admin/migration/types.go     |  70 ++++++++
 pkg/admin/migration/util.go      |  57 +++++++
 3 files changed, 409 insertions(+)
 create mode 100644 pkg/admin/migration/migration.go
 create mode 100644 pkg/admin/migration/types.go
 create mode 100644 pkg/admin/migration/util.go

diff --git a/pkg/admin/migration/migration.go b/pkg/admin/migration/migration.go
new file mode 100644
index 0000000000..6f8e22c85d
--- /dev/null
+++ b/pkg/admin/migration/migration.go
@@ -0,0 +1,282 @@
+package migration
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	crdclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	v1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/client-go/discovery"
+	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/util/retry"
+	apiserviceclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+	apiregistrationv1 "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/typed/apiregistration/v1"
+
+	"k8s.io/klog/v2"
+)
+
+var blackListResources = sets.NewString(
+	"events",
+)
+
+var metadataAccessor = meta.NewAccessor()
+
+type migrator struct {
+	client           dynamic.Interface
+	discoveryClient  discovery.ServerResourcesInterface
+	crdClient        v1.CustomResourceDefinitionInterface
+	apiserviceClient apiregistrationv1.APIServiceInterface
+}
+
+func NewMigrator(kubeConfigPath string) (*migrator, error) {
+	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
+	if err != nil {
+		return nil, err
+	}
+	clientset, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return nil, err
+	}
+	crd, err := crdclient.NewForConfig(restConfig)
+	if err != nil {
+		return nil, err
+	}
+	apiservice, err := apiserviceclient.NewForConfig(restConfig)
+	if err != nil {
+		return nil, err
+	}
+	dynamic, err := dynamic.NewForConfig(restConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	return &migrator{
+		client:           dynamic,
+		discoveryClient:  clientset.Discovery(),
+		crdClient:        crd.ApiextensionsV1().CustomResourceDefinitions(),
+		apiserviceClient: apiservice.ApiregistrationV1().APIServices(),
+	}, nil
+}
+
+func (d *migrator) Start(ctx context.Context) (*MigrationResultList, error) {
+	return d.start(ctx)
+}
+
+func (d *migrator) start(ctx context.Context) (*MigrationResultList, error) {
+	schemas, err := d.findMigratableResources(ctx)
+	if err != nil {
+		return nil, err
+	}
+	results := &MigrationResultList{
+		Status: MigrationSuccess,
+	}
+	errorOccured := false
+	start := time.Now()
+	klog.Info("schema migration started")
+
+	wg := sync.WaitGroup{}
+	lock := sync.Mutex{}
+	wg.Add(len(schemas))
+	for _, sch := range schemas {
+		go func(wg *sync.WaitGroup, sch schema.GroupVersionResource) {
+			defer wg.Done()
+			objectList := &unstructured.UnstructuredList{}
+			var migrationErr error
+
+			migrationErr = retry.OnError(retry.DefaultBackoff, canRetry, func() error {
+				objectList, migrationErr = d.list(ctx, sch, metav1.ListOptions{})
+				if migrationErr != nil {
+					return migrationErr
+				}
+				return nil
+			})
+
+			if migrationErr != nil {
+				errorOccured = true
+				migrationErr = fmt.Errorf("could not list resources: %+v", migrationErr)
+				lock.Lock()
+				results.Items = append(results.Items, MigrationResult{Error: migrationErr, ResourceVersion: sch, Timestamp: time.Now()})
+				lock.Unlock()
+				return
+			}
+
+			for _, object := range objectList.Items {
+				ref := object
+				migrationErr := d.migrateOneItem(ctx, sch, &ref)
+				if migrationErr != nil {
+					errorOccured = true
+				}
+				lock.Lock()
+				results.Items = append(results.Items, MigrationResult{
+					Error:           migrationErr,
+					ResourceVersion: sch,
+					ObjectMeta:      getObjectMeta(&ref),
+					Timestamp:       time.Now()})
+				lock.Unlock()
+			}
+		}(&wg, sch)
+	}
+	if errorOccured {
+		results.Status = MigrationFailure
+	}
+	wg.Wait()
+	klog.Infof("schema migration finished, it took %s to complete", time.Since(start).String())
+	return results, nil
+}
+
+func (d *migrator) findMigratableResources(ctx context.Context) ([]schema.GroupVersionResource, error) {
+	customGroups, err := d.findCustomGroups(ctx)
+	if err != nil {
+		return nil, err
+	}
+	aggregatedGroups, err := d.findAggregatedGroups(ctx)
+	if err != nil {
+		return nil, err
+	}
+	resourceToGroupVersions := make(map[string][]schema.GroupVersion)
+	_, resourceLists, err := d.discoveryClient.ServerGroupsAndResources()
+	if err != nil {
+		return nil, err
+	}
+	for _, resourceList := range resourceLists {
+		gv, err := schema.ParseGroupVersion(resourceList.GroupVersion)
+		if err != nil {
+			klog.Errorf("cannot parse group version %s, ignored", resourceList.GroupVersion)
+			continue
+		}
+		if customGroups.Has(gv.Group) {
+			klog.V(4).Infof("ignored group %v because it's a custom group", gv.Group)
+			continue
+		}
+		if aggregatedGroups.Has(gv.Group) {
+			klog.V(4).Infof("ignored group %v because it's an aggregated group", gv.Group)
+			continue
+		}
+		for _, r := range resourceList.APIResources {
+			// ignore subresources
+			if strings.Contains(r.Name, "/") {
+				continue
+			}
+			if blackListResources.Has(r.Name) {
+				continue
+			}
+			// ignore resources that cannot be listed and updated
+			if !sets.NewString(r.Verbs...).HasAll("list", "update") {
+				continue
+			}
+			gvs := resourceToGroupVersions[r.Name]
+			gvs = append(gvs, gv)
+			resourceToGroupVersions[r.Name] = gvs
+		}
+	}
+
+	ret := []schema.GroupVersionResource{}
+	for resource, groupVersions := range resourceToGroupVersions {
+		if len(groupVersions) == 1 {
+			continue
+		}
+		ret = append(ret, groupVersions[0].WithResource(resource))
+	}
+	return ret, nil
+}
+
+func (m *migrator) migrateOneItem(ctx context.Context, resource schema.GroupVersionResource, item *unstructured.Unstructured) error {
+	namespace, err := metadataAccessor.Namespace(item)
+	if err != nil {
+		return err
+	}
+	name, err := metadataAccessor.Name(item)
+	if err != nil {
+		return err
+	}
+	getBeforePut := false
+	for {
+		getBeforePut, err = m.try(ctx, resource, namespace, name, item, getBeforePut)
+		if err == nil || errors.IsNotFound(err) {
+			return nil
+		}
+		if canRetry(err) {
+			seconds, delay := errors.SuggestsClientDelay(err)
+			switch {
+			case delay && len(namespace) > 0:
+				klog.Warningf("migration of %s, in the %s namespace, will be retried after a %ds delay: %v", name, namespace, seconds, err)
+				time.Sleep(time.Duration(seconds) * time.Second)
+			case delay:
+				klog.Warningf("migration of %s will be retried after a %ds delay: %v", name, seconds, err)
+				time.Sleep(time.Duration(seconds) * time.Second)
+			case !delay && len(namespace) > 0:
+				klog.Warningf("migration of %s, in the %s namespace, will be retried: %v", name, namespace, err)
+			default:
+				klog.Warningf("migration of %s will be retried: %v", name, err)
+			}
+			continue
+		}
+		// error is not retriable
+		return fmt.Errorf("can not retry: %+v", err)
+	}
+}
+
+func (m *migrator) try(ctx context.Context, resource schema.GroupVersionResource, namespace, name string, item *unstructured.Unstructured, get bool) (bool, error) {
+	var err error
+	if get {
+		item, err = m.client.
+			Resource(resource).
+			Namespace(namespace).
+			Get(ctx, name, metav1.GetOptions{})
+		if err != nil {
+			return true, err
+		}
+	}
+	_, err = m.client.
+		Resource(resource).
+		Namespace(namespace).
+		Update(ctx, item, metav1.UpdateOptions{})
+	if err == nil {
+		return false, nil
+	}
+	return errors.IsConflict(err), err
+}
+
+func (m *migrator) list(ctx context.Context, resource schema.GroupVersionResource, options metav1.ListOptions) (*unstructured.UnstructuredList, error) {
+	return m.client.
+		Resource(resource).
+		Namespace(metav1.NamespaceAll).
+		List(ctx, options)
+}
+
+func (d *migrator) findCustomGroups(ctx context.Context) (sets.Set[string], error) {
+	ret := sets.New[string]()
+	l, err := d.crdClient.List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return ret, err
+	}
+	for _, crd := range l.Items {
+		ret.Insert(crd.Spec.Group)
+	}
+	return ret, nil
+}
+
+func (d *migrator) findAggregatedGroups(ctx context.Context) (sets.Set[string], error) {
+	ret := sets.New[string]()
+	l, err := d.apiserviceClient.List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return ret, err
+	}
+	for _, apiservice := range l.Items {
+		if apiservice.Spec.Service != nil {
+			ret.Insert(apiservice.Spec.Group)
+		}
+	}
+	return ret, nil
+}
diff --git a/pkg/admin/migration/types.go b/pkg/admin/migration/types.go
new file mode 100644
index 0000000000..29e9e96e1a
--- /dev/null
+++ b/pkg/admin/migration/types.go
@@ -0,0 +1,70 @@
+package migration
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type MigratonStatus string
+
+const (
+	MigrationSuccess MigratonStatus = "success"
+	MigrationFailure MigratonStatus = "failure"
+)
+
+// Container for individual migration attempts
+type MigrationResult struct {
+	Error           error                       `json:"error,omitempty"`
+	ResourceVersion schema.GroupVersionResource `json:"resourceVersion"`
+	Timestamp       time.Time                   `json:"timestamp"`
+	ObjectMeta      *metav1.ObjectMeta          `json:"object,omitempty"`
+}
+
+type MigrationResultList struct {
+	Status MigratonStatus
+	Items  []MigrationResult
+}
+
+func (m MigrationResultList) String() string {
+	buffer := strings.Builder{}
+	for _, result := range m.Items {
+		objectInfo := result.ResourceVersion.String()
+		if result.ObjectMeta != nil {
+			objectInfo = fmt.Sprintf("%s Namespace=%s Name=%s", objectInfo, result.ObjectMeta.GetNamespace(), result.ObjectMeta.GetName())
+		}
+
+		info := fmt.Sprintf("%s MigrationStatus=%s %s\n", result.Timestamp.String(), MigrationSuccess, objectInfo)
+		if result.Error != nil {
+			info = fmt.Sprintf("%s MigrationStatus=%s %s : %v\n", result.Timestamp.String(), MigrationFailure, objectInfo, result.Error)
+		}
+		buffer.WriteString(info)
+	}
+	return buffer.String()
+}
+
+func (m MigrationResultList) Bytes() []byte {
+	return []byte(m.String())
+}
+
+type ErrRetriable struct {
+	error
+}
+
+func (ErrRetriable) Temporary() bool { return true }
+
+type ErrNotRetriable struct {
+	error
+}
+
+func (ErrNotRetriable) Temporary() bool { return false }
+
+// TemporaryError is a wrapper interface that is used to determine if an error can be retried.
+type TemporaryError interface {
+	error
+	// Temporary should return true if this is a temporary error
+	Temporary() bool
+}
diff --git a/pkg/admin/migration/util.go b/pkg/admin/migration/util.go
new file mode 100644
index 0000000000..090cdb8545
--- /dev/null
+++ b/pkg/admin/migration/util.go
@@ -0,0 +1,57 @@
+package migration
+
+import (
+	"strings"
+
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/net"
+)
+
+func getObjectMeta(item *unstructured.Unstructured) *metav1.ObjectMeta {
+	return &metav1.ObjectMeta{
+		Name:      item.GetName(),
+		Namespace: item.GetNamespace(),
+	}
+}
+
+func canRetry(err error) bool {
+	err = interpret(err)
+	if temp, ok := err.(TemporaryError); ok && !temp.Temporary() {
+		return false
+	}
+	return true
+}
+
+func interpret(err error) error {
+	switch {
+	case err == nil:
+		return nil
+	case errors.IsNotFound(err):
+		// if the object is deleted, there is no need to migrate
+		return nil
+	case errors.IsMethodNotSupported(err):
+		return ErrNotRetriable{err}
+	case errors.IsConflict(err):
+		return ErrRetriable{err}
+	case errors.IsServerTimeout(err):
+		return ErrRetriable{err}
+	case errors.IsTooManyRequests(err):
+		return ErrRetriable{err}
+	case net.IsProbableEOF(err):
+		return ErrRetriable{err}
+	case net.IsConnectionReset(err):
+		return ErrRetriable{err}
+	case net.IsNoRoutesError(err):
+		return ErrRetriable{err}
+	case isConnectionRefusedError(err):
+		return ErrRetriable{err}
+	default:
+		return err
+	}
+}
+
+func isConnectionRefusedError(err error) bool {
+	return strings.Contains(err.Error(), "connection refused")
+}

From fe450aa7384dd6cebd7a0fb4f8b6b8a1c1844df5 Mon Sep 17 00:00:00 2001
From: ehila <ehila@redhat.com>
Date: Mon, 26 Jun 2023 13:56:03 -0400
Subject: [PATCH 2/8] upkeep: updated to include suggested changes

Signed-off-by: ehila <ehila@redhat.com>
---
 pkg/admin/migration/migration.go | 80 ++++++++++++++++----------------
 pkg/admin/migration/types.go     |  8 ++--
 pkg/admin/migration/util.go      |  6 +--
 3 files changed, 47 insertions(+), 47 deletions(-)

diff --git a/pkg/admin/migration/migration.go b/pkg/admin/migration/migration.go
index 6f8e22c85d..04175d82ee 100644
--- a/pkg/admin/migration/migration.go
+++ b/pkg/admin/migration/migration.go
@@ -26,7 +26,7 @@ import (
 	"k8s.io/klog/v2"
 )
 
-var blackListResources = sets.NewString(
+var excludeResources = sets.NewString(
 	"events",
 )
 
@@ -42,23 +42,23 @@ type migrator struct {
 func NewMigrator(kubeConfigPath string) (*migrator, error) {
 	restConfig, err := clientcmd.BuildConfigFromFlags("", kubeConfigPath)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to build rest config: %w", err)
 	}
 	clientset, err := kubernetes.NewForConfig(restConfig)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to build kubernetes clientset config: %w", err)
 	}
 	crd, err := crdclient.NewForConfig(restConfig)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to build crd clientset config: %w", err)
 	}
 	apiservice, err := apiserviceclient.NewForConfig(restConfig)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to build apiservice client config: %w", err)
 	}
 	dynamic, err := dynamic.NewForConfig(restConfig)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to build dynamic client config: %w", err)
 	}
 
 	return &migrator{
@@ -92,12 +92,12 @@ func (d *migrator) start(ctx context.Context) (*MigrationResultList, error) {
 		go func(wg *sync.WaitGroup, sch schema.GroupVersionResource) {
 			defer wg.Done()
 			objectList := &unstructured.UnstructuredList{}
-			var migrationErr error
 
-			migrationErr = retry.OnError(retry.DefaultBackoff, canRetry, func() error {
-				objectList, migrationErr = d.list(ctx, sch, metav1.ListOptions{})
-				if migrationErr != nil {
-					return migrationErr
+			migrationErr := retry.OnError(retry.DefaultBackoff, canRetry, func() error {
+				var err error
+				objectList, err = d.client.Resource(sch).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
+				if err != nil {
+					return err
 				}
 				return nil
 			})
@@ -106,35 +106,49 @@ func (d *migrator) start(ctx context.Context) (*MigrationResultList, error) {
 				errorOccured = true
 				migrationErr = fmt.Errorf("could not list resources: %+v", migrationErr)
 				lock.Lock()
+				defer lock.Unlock()
 				results.Items = append(results.Items, MigrationResult{Error: migrationErr, ResourceVersion: sch, Timestamp: time.Now()})
-				lock.Unlock()
 				return
 			}
 
+			localResults := &MigrationResultList{Status: MigrationSuccess}
 			for _, object := range objectList.Items {
 				ref := object
 				migrationErr := d.migrateOneItem(ctx, sch, &ref)
 				if migrationErr != nil {
 					errorOccured = true
 				}
-				lock.Lock()
-				results.Items = append(results.Items, MigrationResult{
+				localResults.Items = append(localResults.Items, MigrationResult{
 					Error:           migrationErr,
 					ResourceVersion: sch,
-					ObjectMeta:      getObjectMeta(&ref),
+					NamespacedName:  getNamespacedName(&ref),
 					Timestamp:       time.Now()})
-				lock.Unlock()
 			}
+
+			lock.Lock()
+			defer lock.Unlock()
+			results.Items = append(results.Items, localResults.Items...)
 		}(&wg, sch)
 	}
+	wg.Wait()
 	if errorOccured {
 		results.Status = MigrationFailure
 	}
-	wg.Wait()
-	klog.Infof("schema migration finished, it took %s to complete", time.Since(start).String())
+	klog.InfoS("schema migration finished", "duration", time.Since(start).String())
 	return results, nil
 }
 
+// findMigratableResources finds all the resources that potentially need
+// migration. Although all migratable resources are accessible via multiple
+// versions, the returned list only include one version.
+//
+// It builds the list in these steps:
+// 1. build a map from resource name to the groupVersions, excluding subresources, custom resources, or aggregated resources.
+// 2. exclude all the resource that is only available from one groupVersions.
+// 3. exclude the resource that does not support "list" and "update" (thus not migratable).
+//
+// More information can be found here:
+// https://github.com/kubernetes-sigs/kube-storage-version-migrator/blob/acdee30ced218b79e39c6a701985e8cd8bd33824/pkg/initializer/discover.go#L55-L125
 func (d *migrator) findMigratableResources(ctx context.Context) ([]schema.GroupVersionResource, error) {
 	customGroups, err := d.findCustomGroups(ctx)
 	if err != nil {
@@ -152,15 +166,15 @@ func (d *migrator) findMigratableResources(ctx context.Context) ([]schema.GroupV
 	for _, resourceList := range resourceLists {
 		gv, err := schema.ParseGroupVersion(resourceList.GroupVersion)
 		if err != nil {
-			klog.Errorf("cannot parse group version %s, ignored", resourceList.GroupVersion)
+			klog.ErrorS(err, "cannot parse group version, ignored", "groupVersion", resourceList.GroupVersion)
 			continue
 		}
 		if customGroups.Has(gv.Group) {
-			klog.V(4).Infof("ignored group %v because it's a custom group", gv.Group)
+			klog.InfoS("ignored because it's a custom group", "group", gv.Group)
 			continue
 		}
 		if aggregatedGroups.Has(gv.Group) {
-			klog.V(4).Infof("ignored group %v because it's an aggregated group", gv.Group)
+			klog.InfoS("ignored because it's an aggregated group", "group", gv.Group)
 			continue
 		}
 		for _, r := range resourceList.APIResources {
@@ -168,7 +182,8 @@ func (d *migrator) findMigratableResources(ctx context.Context) ([]schema.GroupV
 			if strings.Contains(r.Name, "/") {
 				continue
 			}
-			if blackListResources.Has(r.Name) {
+			// ignore excluded resources
+			if excludeResources.Has(r.Name) {
 				continue
 			}
 			// ignore resources that cannot be listed and updated
@@ -183,6 +198,8 @@ func (d *migrator) findMigratableResources(ctx context.Context) ([]schema.GroupV
 
 	ret := []schema.GroupVersionResource{}
 	for resource, groupVersions := range resourceToGroupVersions {
+		// if a resource only has one version, no migration is required
+		// resources that have more than one version are eligible for migration.
 		if len(groupVersions) == 1 {
 			continue
 		}
@@ -208,17 +225,9 @@ func (m *migrator) migrateOneItem(ctx context.Context, resource schema.GroupVers
 		}
 		if canRetry(err) {
 			seconds, delay := errors.SuggestsClientDelay(err)
-			switch {
-			case delay && len(namespace) > 0:
-				klog.Warningf("migration of %s, in the %s namespace, will be retried after a %ds delay: %v", name, namespace, seconds, err)
-				time.Sleep(time.Duration(seconds) * time.Second)
-			case delay:
-				klog.Warningf("migration of %s will be retried after a %ds delay: %v", name, seconds, err)
+			klog.ErrorS(err, "migration of an object will be retried", "name", name, "namespace", namespace, "delay", seconds)
+			if delay {
 				time.Sleep(time.Duration(seconds) * time.Second)
-			case !delay && len(namespace) > 0:
-				klog.Warningf("migration of %s, in the %s namespace, will be retried: %v", name, namespace, err)
-			default:
-				klog.Warningf("migration of %s will be retried: %v", name, err)
 			}
 			continue
 		}
@@ -248,13 +257,6 @@ func (m *migrator) try(ctx context.Context, resource schema.GroupVersionResource
 	return errors.IsConflict(err), err
 }
 
-func (m *migrator) list(ctx context.Context, resource schema.GroupVersionResource, options metav1.ListOptions) (*unstructured.UnstructuredList, error) {
-	return m.client.
-		Resource(resource).
-		Namespace(metav1.NamespaceAll).
-		List(ctx, options)
-}
-
 func (d *migrator) findCustomGroups(ctx context.Context) (sets.Set[string], error) {
 	ret := sets.New[string]()
 	l, err := d.crdClient.List(ctx, metav1.ListOptions{})
diff --git a/pkg/admin/migration/types.go b/pkg/admin/migration/types.go
index 29e9e96e1a..a8b290ba4c 100644
--- a/pkg/admin/migration/types.go
+++ b/pkg/admin/migration/types.go
@@ -5,8 +5,8 @@ import (
 	"strings"
 	"time"
 
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	apitypes "k8s.io/apimachinery/pkg/types"
 )
 
 type MigratonStatus string
@@ -21,7 +21,7 @@ type MigrationResult struct {
 	Error           error                       `json:"error,omitempty"`
 	ResourceVersion schema.GroupVersionResource `json:"resourceVersion"`
 	Timestamp       time.Time                   `json:"timestamp"`
-	ObjectMeta      *metav1.ObjectMeta          `json:"object,omitempty"`
+	NamespacedName  apitypes.NamespacedName     `json:"namespacedName,omitempty"`
 }
 
 type MigrationResultList struct {
@@ -33,9 +33,7 @@ func (m MigrationResultList) String() string {
 	buffer := strings.Builder{}
 	for _, result := range m.Items {
 		objectInfo := result.ResourceVersion.String()
-		if result.ObjectMeta != nil {
-			objectInfo = fmt.Sprintf("%s Namespace=%s Name=%s", objectInfo, result.ObjectMeta.GetNamespace(), result.ObjectMeta.GetName())
-		}
+		objectInfo = fmt.Sprintf("%s Namespace=%s Name=%s", objectInfo, result.NamespacedName.Namespace, result.NamespacedName.Name)
 
 		info := fmt.Sprintf("%s MigrationStatus=%s %s\n", result.Timestamp.String(), MigrationSuccess, objectInfo)
 		if result.Error != nil {
diff --git a/pkg/admin/migration/util.go b/pkg/admin/migration/util.go
index 090cdb8545..a0111cbe3a 100644
--- a/pkg/admin/migration/util.go
+++ b/pkg/admin/migration/util.go
@@ -4,13 +4,13 @@ import (
 	"strings"
 
 	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	apitypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/net"
 )
 
-func getObjectMeta(item *unstructured.Unstructured) *metav1.ObjectMeta {
-	return &metav1.ObjectMeta{
+func getNamespacedName(item *unstructured.Unstructured) apitypes.NamespacedName {
+	return apitypes.NamespacedName{
 		Name:      item.GetName(),
 		Namespace: item.GetNamespace(),
 	}

From aa46030e860924fae59f5787b5534fe054848a2c Mon Sep 17 00:00:00 2001
From: ehila <ehila@redhat.com>
Date: Mon, 26 Jun 2023 18:01:17 -0400
Subject: [PATCH 3/8] upkeep: remove concurrency

removing for now to remove complexity, no clear performance gain is present
should revisit if performance issues crop up

Signed-off-by: ehila <ehila@redhat.com>
---
 pkg/admin/migration/migration.go | 66 +++++++++++++-------------------
 1 file changed, 26 insertions(+), 40 deletions(-)

diff --git a/pkg/admin/migration/migration.go b/pkg/admin/migration/migration.go
index 04175d82ee..787388d614 100644
--- a/pkg/admin/migration/migration.go
+++ b/pkg/admin/migration/migration.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"strings"
-	"sync"
 	"time"
 
 	crdclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
@@ -85,52 +84,38 @@ func (d *migrator) start(ctx context.Context) (*MigrationResultList, error) {
 	start := time.Now()
 	klog.Info("schema migration started")
 
-	wg := sync.WaitGroup{}
-	lock := sync.Mutex{}
-	wg.Add(len(schemas))
 	for _, sch := range schemas {
-		go func(wg *sync.WaitGroup, sch schema.GroupVersionResource) {
-			defer wg.Done()
-			objectList := &unstructured.UnstructuredList{}
+		objectList := &unstructured.UnstructuredList{}
 
-			migrationErr := retry.OnError(retry.DefaultBackoff, canRetry, func() error {
-				var err error
-				objectList, err = d.client.Resource(sch).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
-				if err != nil {
-					return err
-				}
-				return nil
-			})
+		migrationErr := retry.OnError(retry.DefaultBackoff, canRetry, func() error {
+			var err error
+			objectList, err = d.client.Resource(sch).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
+			if err != nil {
+				return err
+			}
+			return nil
+		})
+
+		if migrationErr != nil {
+			errorOccured = true
+			migrationErr = fmt.Errorf("could not list resources: %+v", migrationErr)
+			results.Items = append(results.Items, MigrationResult{Error: migrationErr, ResourceVersion: sch, Timestamp: time.Now()})
+			continue
+		}
 
+		for _, object := range objectList.Items {
+			ref := object
+			migrationErr := d.migrateOneItem(ctx, sch, &ref)
 			if migrationErr != nil {
 				errorOccured = true
-				migrationErr = fmt.Errorf("could not list resources: %+v", migrationErr)
-				lock.Lock()
-				defer lock.Unlock()
-				results.Items = append(results.Items, MigrationResult{Error: migrationErr, ResourceVersion: sch, Timestamp: time.Now()})
-				return
 			}
-
-			localResults := &MigrationResultList{Status: MigrationSuccess}
-			for _, object := range objectList.Items {
-				ref := object
-				migrationErr := d.migrateOneItem(ctx, sch, &ref)
-				if migrationErr != nil {
-					errorOccured = true
-				}
-				localResults.Items = append(localResults.Items, MigrationResult{
-					Error:           migrationErr,
-					ResourceVersion: sch,
-					NamespacedName:  getNamespacedName(&ref),
-					Timestamp:       time.Now()})
-			}
-
-			lock.Lock()
-			defer lock.Unlock()
-			results.Items = append(results.Items, localResults.Items...)
-		}(&wg, sch)
+			results.Items = append(results.Items, MigrationResult{
+				Error:           migrationErr,
+				ResourceVersion: sch,
+				NamespacedName:  getNamespacedName(&ref),
+				Timestamp:       time.Now()})
+		}
 	}
-	wg.Wait()
 	if errorOccured {
 		results.Status = MigrationFailure
 	}
@@ -221,6 +206,7 @@ func (m *migrator) migrateOneItem(ctx context.Context, resource schema.GroupVers
 	for {
 		getBeforePut, err = m.try(ctx, resource, namespace, name, item, getBeforePut)
 		if err == nil || errors.IsNotFound(err) {
+			klog.InfoS("successfully migrated object", "name", name, "namespace", namespace, "resource", resource.String())
 			return nil
 		}
 		if canRetry(err) {

From afd6c80b95e561d001867f467b6550f0f34fefd3 Mon Sep 17 00:00:00 2001
From: ehila <ehila@redhat.com>
Date: Thu, 29 Jun 2023 08:12:41 -0400
Subject: [PATCH 4/8] feat: add continue token check for list

Signed-off-by: ehila <ehila@redhat.com>
---
 pkg/admin/migration/migration.go | 77 ++++++++++++++++++++++----------
 pkg/admin/migration/util.go      | 14 ++++++
 2 files changed, 68 insertions(+), 23 deletions(-)

diff --git a/pkg/admin/migration/migration.go b/pkg/admin/migration/migration.go
index 787388d614..9572e3829d 100644
--- a/pkg/admin/migration/migration.go
+++ b/pkg/admin/migration/migration.go
@@ -84,36 +84,67 @@ func (d *migrator) start(ctx context.Context) (*MigrationResultList, error) {
 	start := time.Now()
 	klog.Info("schema migration started")
 
+	// Currenlty we are sequentially migrating items, we will need to revisit this if performance becomes a problem
 	for _, sch := range schemas {
-		objectList := &unstructured.UnstructuredList{}
+		// A list of objects might be very large, they will be chunked results with a continue token
+		// here we loop for as many times we have a continue token or an error occured
+		continueToken := ""
+		for {
+			objectList := &unstructured.UnstructuredList{}
+			migrationErr := retry.OnError(retry.DefaultBackoff, canRetry, func() error {
+				var err error
+				objectList, err = d.client.Resource(sch).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{
+					Continue: continueToken,
+				})
+				if err != nil {
+					return err
+				}
+				return nil
+			})
 
-		migrationErr := retry.OnError(retry.DefaultBackoff, canRetry, func() error {
-			var err error
-			objectList, err = d.client.Resource(sch).Namespace(metav1.NamespaceAll).List(ctx, metav1.ListOptions{})
-			if err != nil {
-				return err
+			// If resource expired error, retry
+			if migrationErr != nil && errors.IsResourceExpired(migrationErr) {
+				token, err := inconsistentContinueToken(migrationErr)
+				if err != nil {
+					err = fmt.Errorf("failed to get continue token: %w", err)
+					results.Items = append(results.Items, MigrationResult{Error: err, ResourceVersion: sch, Timestamp: time.Now()})
+					break
+				}
+				continueToken = token
+				continue
 			}
-			return nil
-		})
-
-		if migrationErr != nil {
-			errorOccured = true
-			migrationErr = fmt.Errorf("could not list resources: %+v", migrationErr)
-			results.Items = append(results.Items, MigrationResult{Error: migrationErr, ResourceVersion: sch, Timestamp: time.Now()})
-			continue
-		}
 
-		for _, object := range objectList.Items {
-			ref := object
-			migrationErr := d.migrateOneItem(ctx, sch, &ref)
 			if migrationErr != nil {
 				errorOccured = true
+				migrationErr = fmt.Errorf("could not list resources: %w", migrationErr)
+				results.Items = append(results.Items, MigrationResult{Error: migrationErr, ResourceVersion: sch, Timestamp: time.Now()})
+				break
+			}
+
+			for _, object := range objectList.Items {
+				ref := object
+				migrationErr := d.migrateOneItem(ctx, sch, &ref)
+				if migrationErr != nil {
+					errorOccured = true
+				}
+				results.Items = append(results.Items, MigrationResult{
+					Error:           migrationErr,
+					ResourceVersion: sch,
+					NamespacedName:  getNamespacedName(&ref),
+					Timestamp:       time.Now()})
+			}
+
+			// Check if the list contains a continue token
+			token, err := metadataAccessor.Continue(objectList)
+			if err != nil {
+				err = fmt.Errorf("failed to get continue token: %w", err)
+				results.Items = append(results.Items, MigrationResult{Error: err, ResourceVersion: sch, Timestamp: time.Now()})
+				break
+			}
+			if len(token) == 0 {
+				break
 			}
-			results.Items = append(results.Items, MigrationResult{
-				Error:           migrationErr,
-				ResourceVersion: sch,
-				NamespacedName:  getNamespacedName(&ref),
-				Timestamp:       time.Now()})
+			continueToken = token
 		}
 	}
 	if errorOccured {
diff --git a/pkg/admin/migration/util.go b/pkg/admin/migration/util.go
index a0111cbe3a..444e852405 100644
--- a/pkg/admin/migration/util.go
+++ b/pkg/admin/migration/util.go
@@ -1,6 +1,8 @@
 package migration
 
 import (
+	"fmt"
+	"reflect"
 	"strings"
 
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -55,3 +57,15 @@ func interpret(err error) error {
 func isConnectionRefusedError(err error) bool {
 	return strings.Contains(err.Error(), "connection refused")
 }
+
+func inconsistentContinueToken(err error) (string, error) {
+	status, ok := err.(errors.APIStatus)
+	if !ok {
+		return "", fmt.Errorf("expected error to implement the APIStatus interface, got %v", reflect.TypeOf(err))
+	}
+	token := status.Status().ListMeta.Continue
+	if len(token) == 0 {
+		return "", fmt.Errorf("expected non empty continue token")
+	}
+	return token, nil
+}

From 0886da1c8ce1e15cc70f1e480b2213a418c9c23e Mon Sep 17 00:00:00 2001
From: ehila <ehila@redhat.com>
Date: Fri, 30 Jun 2023 12:22:36 -0400
Subject: [PATCH 5/8] fix: golangci linter errors

Signed-off-by: ehila <ehila@redhat.com>
---
 pkg/admin/migration/migration.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/admin/migration/migration.go b/pkg/admin/migration/migration.go
index 9572e3829d..607f4539cd 100644
--- a/pkg/admin/migration/migration.go
+++ b/pkg/admin/migration/migration.go
@@ -84,10 +84,10 @@ func (d *migrator) start(ctx context.Context) (*MigrationResultList, error) {
 	start := time.Now()
 	klog.Info("schema migration started")
 
-	// Currenlty we are sequentially migrating items, we will need to revisit this if performance becomes a problem
+	// Currently we are sequentially migrating items, we will need to revisit this if performance becomes a problem
 	for _, sch := range schemas {
 		// A list of objects might be very large, they will be chunked results with a continue token
-		// here we loop for as many times we have a continue token or an error occured
+		// here we loop for as many times we have a continue token or an error occurred
 		continueToken := ""
 		for {
 			objectList := &unstructured.UnstructuredList{}

From 326d890526b21eacb529aa5a3f40eec129b4d3c8 Mon Sep 17 00:00:00 2001
From: ehila <ehila@redhat.com>
Date: Thu, 6 Jul 2023 09:54:45 -0400
Subject: [PATCH 6/8] feat: change to gather logic for resources

updated gather logic to be more inline with trigger controller in kube storage migrator

Signed-off-by: ehila <ehila@redhat.com>
---
 pkg/admin/migration/migration.go | 95 ++++++++++++--------------------
 1 file changed, 34 insertions(+), 61 deletions(-)

diff --git a/pkg/admin/migration/migration.go b/pkg/admin/migration/migration.go
index 607f4539cd..a7d6a4fe87 100644
--- a/pkg/admin/migration/migration.go
+++ b/pkg/admin/migration/migration.go
@@ -25,6 +25,7 @@ import (
 	"k8s.io/klog/v2"
 )
 
+// Skipping resources which might cycle quickly or cause a lot of overhead to migrate
 var excludeResources = sets.NewString(
 	"events",
 )
@@ -107,7 +108,10 @@ func (d *migrator) start(ctx context.Context) (*MigrationResultList, error) {
 				token, err := inconsistentContinueToken(migrationErr)
 				if err != nil {
 					err = fmt.Errorf("failed to get continue token: %w", err)
-					results.Items = append(results.Items, MigrationResult{Error: err, ResourceVersion: sch, Timestamp: time.Now()})
+					results.Items = append(results.Items, MigrationResult{
+						Error:                err,
+						GroupVersionResource: sch,
+						Timestamp:            time.Now()})
 					break
 				}
 				continueToken = token
@@ -117,28 +121,37 @@ func (d *migrator) start(ctx context.Context) (*MigrationResultList, error) {
 			if migrationErr != nil {
 				errorOccured = true
 				migrationErr = fmt.Errorf("could not list resources: %w", migrationErr)
-				results.Items = append(results.Items, MigrationResult{Error: migrationErr, ResourceVersion: sch, Timestamp: time.Now()})
+				results.Items = append(results.Items, MigrationResult{
+					Error:                migrationErr,
+					GroupVersionResource: sch,
+					Timestamp:            time.Now()})
 				break
 			}
 
+			status := MigrationSuccess
 			for _, object := range objectList.Items {
 				ref := object
 				migrationErr := d.migrateOneItem(ctx, sch, &ref)
 				if migrationErr != nil {
 					errorOccured = true
+					status = MigrationFailure
 				}
-				results.Items = append(results.Items, MigrationResult{
-					Error:           migrationErr,
-					ResourceVersion: sch,
-					NamespacedName:  getNamespacedName(&ref),
-					Timestamp:       time.Now()})
 			}
 
+			results.Items = append(results.Items, MigrationResult{
+				Error:                migrationErr,
+				Status:               status,
+				GroupVersionResource: sch,
+				Timestamp:            time.Now()})
+
 			// Check if the list contains a continue token
 			token, err := metadataAccessor.Continue(objectList)
 			if err != nil {
 				err = fmt.Errorf("failed to get continue token: %w", err)
-				results.Items = append(results.Items, MigrationResult{Error: err, ResourceVersion: sch, Timestamp: time.Now()})
+				results.Items = append(results.Items, MigrationResult{
+					Error:                err,
+					GroupVersionResource: sch,
+					Timestamp:            time.Now()})
 				break
 			}
 			if len(token) == 0 {
@@ -166,29 +179,22 @@ func (d *migrator) start(ctx context.Context) (*MigrationResultList, error) {
 // More information can be found here:
 // https://github.com/kubernetes-sigs/kube-storage-version-migrator/blob/acdee30ced218b79e39c6a701985e8cd8bd33824/pkg/initializer/discover.go#L55-L125
 func (d *migrator) findMigratableResources(ctx context.Context) ([]schema.GroupVersionResource, error) {
-	customGroups, err := d.findCustomGroups(ctx)
-	if err != nil {
-		return nil, err
-	}
 	aggregatedGroups, err := d.findAggregatedGroups(ctx)
 	if err != nil {
 		return nil, err
 	}
-	resourceToGroupVersions := make(map[string][]schema.GroupVersion)
-	_, resourceLists, err := d.discoveryClient.ServerGroupsAndResources()
+	ret := []schema.GroupVersionResource{}
+	resourceLists, err := d.discoveryClient.ServerPreferredResources()
 	if err != nil {
 		return nil, err
 	}
 	for _, resourceList := range resourceLists {
 		gv, err := schema.ParseGroupVersion(resourceList.GroupVersion)
 		if err != nil {
-			klog.ErrorS(err, "cannot parse group version, ignored", "groupVersion", resourceList.GroupVersion)
-			continue
-		}
-		if customGroups.Has(gv.Group) {
-			klog.InfoS("ignored because it's a custom group", "group", gv.Group)
+			klog.ErrorS(err, "cannot parse group version, ignored", "version", resourceList.GroupVersion)
 			continue
 		}
+
 		if aggregatedGroups.Has(gv.Group) {
 			klog.InfoS("ignored because it's an aggregated group", "group", gv.Group)
 			continue
@@ -196,31 +202,23 @@ func (d *migrator) findMigratableResources(ctx context.Context) ([]schema.GroupV
 		for _, r := range resourceList.APIResources {
 			// ignore subresources
 			if strings.Contains(r.Name, "/") {
+				klog.InfoS("ignored subresource", "group", gv.Group, "name", r.Name, "version", gv.Version)
 				continue
 			}
 			// ignore excluded resources
 			if excludeResources.Has(r.Name) {
+				klog.InfoS("ignored excluded resource", "group", gv.Group, "name", r.Name, "version", gv.Version)
 				continue
 			}
 			// ignore resources that cannot be listed and updated
 			if !sets.NewString(r.Verbs...).HasAll("list", "update") {
+				klog.InfoS("ignored because verb does not contain list or update", "group", gv.Group, "name", r.Name, "version", gv.Version)
 				continue
 			}
-			gvs := resourceToGroupVersions[r.Name]
-			gvs = append(gvs, gv)
-			resourceToGroupVersions[r.Name] = gvs
+			ret = append(ret, gv.WithResource(r.Name))
 		}
 	}
 
-	ret := []schema.GroupVersionResource{}
-	for resource, groupVersions := range resourceToGroupVersions {
-		// if a resource only has one version, no migration is required
-		// resources that have more than one version are eligible for migration.
-		if len(groupVersions) == 1 {
-			continue
-		}
-		ret = append(ret, groupVersions[0].WithResource(resource))
-	}
 	return ret, nil
 }
 
@@ -233,9 +231,9 @@ func (m *migrator) migrateOneItem(ctx context.Context, resource schema.GroupVers
 	if err != nil {
 		return err
 	}
-	getBeforePut := false
+
 	for {
-		getBeforePut, err = m.try(ctx, resource, namespace, name, item, getBeforePut)
+		err = m.try(ctx, resource, namespace, name, item)
 		if err == nil || errors.IsNotFound(err) {
 			klog.InfoS("successfully migrated object", "name", name, "namespace", namespace, "resource", resource.String())
 			return nil
@@ -253,37 +251,12 @@ func (m *migrator) migrateOneItem(ctx context.Context, resource schema.GroupVers
 	}
 }
 
-func (m *migrator) try(ctx context.Context, resource schema.GroupVersionResource, namespace, name string, item *unstructured.Unstructured, get bool) (bool, error) {
-	var err error
-	if get {
-		item, err = m.client.
-			Resource(resource).
-			Namespace(namespace).
-			Get(ctx, name, metav1.GetOptions{})
-		if err != nil {
-			return true, err
-		}
-	}
-	_, err = m.client.
+func (m *migrator) try(ctx context.Context, resource schema.GroupVersionResource, namespace, name string, item *unstructured.Unstructured) error {
+	_, err := m.client.
 		Resource(resource).
 		Namespace(namespace).
 		Update(ctx, item, metav1.UpdateOptions{})
-	if err == nil {
-		return false, nil
-	}
-	return errors.IsConflict(err), err
-}
-
-func (d *migrator) findCustomGroups(ctx context.Context) (sets.Set[string], error) {
-	ret := sets.New[string]()
-	l, err := d.crdClient.List(ctx, metav1.ListOptions{})
-	if err != nil {
-		return ret, err
-	}
-	for _, crd := range l.Items {
-		ret.Insert(crd.Spec.Group)
-	}
-	return ret, nil
+	return err
 }
 
 func (d *migrator) findAggregatedGroups(ctx context.Context) (sets.Set[string], error) {

From f999ea351880c0a2197b66cd8037f24a521e612a Mon Sep 17 00:00:00 2001
From: ehila <ehila@redhat.com>
Date: Thu, 6 Jul 2023 09:55:45 -0400
Subject: [PATCH 7/8] feat: added call to migration in prerun and output files

put in helper logic to write log and status to backup folder for uvalidation

Signed-off-by: ehila <ehila@redhat.com>
---
 pkg/admin/migration/types.go | 37 ++++++++----------
 pkg/admin/prerun/prerun.go   | 75 +++++++++++++++++++++++++++++++-----
 2 files changed, 81 insertions(+), 31 deletions(-)

diff --git a/pkg/admin/migration/types.go b/pkg/admin/migration/types.go
index a8b290ba4c..6bfde709e1 100644
--- a/pkg/admin/migration/types.go
+++ b/pkg/admin/migration/types.go
@@ -1,12 +1,12 @@
 package migration
 
 import (
+	"encoding/json"
 	"fmt"
-	"strings"
+	"os"
 	"time"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	apitypes "k8s.io/apimachinery/pkg/types"
 )
 
 type MigratonStatus string
@@ -14,14 +14,15 @@ type MigratonStatus string
 const (
 	MigrationSuccess MigratonStatus = "success"
 	MigrationFailure MigratonStatus = "failure"
+	MigrationRunning MigratonStatus = "running"
 )
 
 // Container for individual migration attempts
 type MigrationResult struct {
-	Error           error                       `json:"error,omitempty"`
-	ResourceVersion schema.GroupVersionResource `json:"resourceVersion"`
-	Timestamp       time.Time                   `json:"timestamp"`
-	NamespacedName  apitypes.NamespacedName     `json:"namespacedName,omitempty"`
+	Error     error `json:"Error,omitempty"`
+	Timestamp time.Time
+	Status    MigratonStatus
+	schema.GroupVersionResource
 }
 
 type MigrationResultList struct {
@@ -29,23 +30,17 @@ type MigrationResultList struct {
 	Items  []MigrationResult
 }
 
-func (m MigrationResultList) String() string {
-	buffer := strings.Builder{}
-	for _, result := range m.Items {
-		objectInfo := result.ResourceVersion.String()
-		objectInfo = fmt.Sprintf("%s Namespace=%s Name=%s", objectInfo, result.NamespacedName.Namespace, result.NamespacedName.Name)
-
-		info := fmt.Sprintf("%s MigrationStatus=%s %s\n", result.Timestamp.String(), MigrationSuccess, objectInfo)
-		if result.Error != nil {
-			info = fmt.Sprintf("%s MigrationStatus=%s %s : %v\n", result.Timestamp.String(), MigrationFailure, objectInfo, result.Error)
-		}
-		buffer.WriteString(info)
-	}
-	return buffer.String()
+func (m *MigrationResultList) WriteStatusFile(filePath string) error {
+	data := fmt.Sprintf(`{"Status": "%s"}`, m.Status)
+	return os.WriteFile(filePath, []byte(data), 0644)
 }
 
-func (m MigrationResultList) Bytes() []byte {
-	return []byte(m.String())
+func (m *MigrationResultList) WriteDataFile(filePath string) error {
+	fileData, err := json.Marshal(m.Items)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(filePath, fileData, 0644)
 }
 
 type ErrRetriable struct {
diff --git a/pkg/admin/prerun/prerun.go b/pkg/admin/prerun/prerun.go
index 2241be2a42..e5cf64e5f3 100644
--- a/pkg/admin/prerun/prerun.go
+++ b/pkg/admin/prerun/prerun.go
@@ -2,14 +2,17 @@ package prerun
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 
 	"github.com/openshift/microshift/pkg/admin/data"
+	"github.com/openshift/microshift/pkg/admin/migration"
 	"github.com/openshift/microshift/pkg/config"
 	"github.com/openshift/microshift/pkg/util"
 	"k8s.io/klog/v2"
@@ -17,6 +20,8 @@ import (
 
 var (
 	healthFilepath            = "/var/lib/microshift-backups/health.json"
+	migrationStatusFilepath   = "/var/lib/microshift-backups/migration-status.json"
+	migrationLogFilepath      = "/var/lib/microshift-backups/migration-log.json"
 	errHealthFileDoesNotExist = errors.New("health file does not exist")
 )
 
@@ -189,12 +194,10 @@ func (pr *PreRun) regularPrerun() error {
 
 		if migrationNeeded {
 			_ = migrationNeeded
-			stop, err := runMinimalMicroshift(pr.config)
-			if err != nil {
-				return fmt.Errorf("minimal MicroShift run failed to be ready: %w", err)
+
+			if err := pr.migrate(); err != nil {
+				return fmt.Errorf("failed to migrate resources: %w", err)
 			}
-			defer stop()
-			// TODO: data migration
 
 			if err := writeExecVersionToData(); err != nil {
 				return fmt.Errorf("failed to write MicroShift version to data directory: %w", err)
@@ -210,6 +213,61 @@ func (pr *PreRun) regularPrerun() error {
 	return nil
 }
 
+func (pr *PreRun) migrate() error {
+
+	// Run microshift in minimal mode
+	stop, err := runMinimalMicroshift(pr.config)
+	if err != nil {
+		return fmt.Errorf("minimal MicroShift run failed to be ready: %w", err)
+	}
+	defer stop()
+
+	// Use kubeconfig credentials for updating all resources
+	adminKubeConifg := filepath.Join(pr.config.KubeConfigRootAdminPath(), "kubeconfig")
+	m, err := migration.NewMigrator(adminKubeConifg)
+	if err != nil {
+		return fmt.Errorf("minimal MicroShift run failed to be ready: %w", err)
+	}
+
+	// Begin migration
+
+	// Create status container for results
+	results := &migration.MigrationResultList{
+		Status: migration.MigrationRunning,
+	}
+
+	// Open status file for migration process
+	err = results.WriteStatusFile(migrationStatusFilepath)
+	if err != nil {
+		return fmt.Errorf("failed to write migration status file: %w", err)
+	}
+
+	// Start updating resources
+	results, err = m.Start(context.Background())
+	if err != nil {
+		return fmt.Errorf("migration failed: %w", err)
+	}
+
+	// Write status of over all result
+	err = results.WriteStatusFile(migrationStatusFilepath)
+	if err != nil {
+		return fmt.Errorf("failed to write migration status file: %w", err)
+	}
+
+	// Fail fast if migration failed
+	if results.Status != migration.MigrationSuccess {
+		return fmt.Errorf("migration failed")
+	}
+
+	// Write all migration events as json
+	err = results.WriteDataFile(migrationLogFilepath)
+	if err != nil {
+		return fmt.Errorf("failed to write migration log to file: %w", err)
+	}
+
+	return nil
+}
+
 func (pr *PreRun) upgradeFrom413() error {
 	backupName := data.BackupName("4.13")
 
@@ -217,12 +275,9 @@ func (pr *PreRun) upgradeFrom413() error {
 		return fmt.Errorf("failed to create new backup %q: %w", backupName, err)
 	}
 
-	stop, err := runMinimalMicroshift(pr.config)
-	if err != nil {
-		return fmt.Errorf("minimal MicroShift run failed to be ready: %w", err)
+	if err := pr.migrate(); err != nil {
+		return fmt.Errorf("failed to migrate resources: %w", err)
 	}
-	defer stop()
-	// TODO: data migration
 
 	if err := writeExecVersionToData(); err != nil {
 		return fmt.Errorf("failed to write MicroShift version to data directory: %w", err)

From 09a25561f597e583188e1aa7a2e5c75118e61c8d Mon Sep 17 00:00:00 2001
From: ehila <ehila@redhat.com>
Date: Thu, 6 Jul 2023 11:32:15 -0400
Subject: [PATCH 8/8] fix: linter errors

Signed-off-by: ehila <ehila@redhat.com>
---
 pkg/admin/migration/migration.go | 4 ++--
 pkg/admin/migration/types.go     | 4 ++--
 pkg/admin/migration/util.go      | 9 ---------
 pkg/admin/prerun/prerun.go       | 1 -
 4 files changed, 4 insertions(+), 14 deletions(-)

diff --git a/pkg/admin/migration/migration.go b/pkg/admin/migration/migration.go
index a7d6a4fe87..94c86c17b8 100644
--- a/pkg/admin/migration/migration.go
+++ b/pkg/admin/migration/migration.go
@@ -233,7 +233,7 @@ func (m *migrator) migrateOneItem(ctx context.Context, resource schema.GroupVers
 	}
 
 	for {
-		err = m.try(ctx, resource, namespace, name, item)
+		err = m.try(ctx, resource, namespace, item)
 		if err == nil || errors.IsNotFound(err) {
 			klog.InfoS("successfully migrated object", "name", name, "namespace", namespace, "resource", resource.String())
 			return nil
@@ -251,7 +251,7 @@ func (m *migrator) migrateOneItem(ctx context.Context, resource schema.GroupVers
 	}
 }
 
-func (m *migrator) try(ctx context.Context, resource schema.GroupVersionResource, namespace, name string, item *unstructured.Unstructured) error {
+func (m *migrator) try(ctx context.Context, resource schema.GroupVersionResource, namespace string, item *unstructured.Unstructured) error {
 	_, err := m.client.
 		Resource(resource).
 		Namespace(namespace).
diff --git a/pkg/admin/migration/types.go b/pkg/admin/migration/types.go
index 6bfde709e1..b3618d4e75 100644
--- a/pkg/admin/migration/types.go
+++ b/pkg/admin/migration/types.go
@@ -32,7 +32,7 @@ type MigrationResultList struct {
 
 func (m *MigrationResultList) WriteStatusFile(filePath string) error {
 	data := fmt.Sprintf(`{"Status": "%s"}`, m.Status)
-	return os.WriteFile(filePath, []byte(data), 0644)
+	return os.WriteFile(filePath, []byte(data), 0600)
 }
 
 func (m *MigrationResultList) WriteDataFile(filePath string) error {
@@ -40,7 +40,7 @@ func (m *MigrationResultList) WriteDataFile(filePath string) error {
 	if err != nil {
 		return err
 	}
-	return os.WriteFile(filePath, fileData, 0644)
+	return os.WriteFile(filePath, fileData, 0600)
 }
 
 type ErrRetriable struct {
diff --git a/pkg/admin/migration/util.go b/pkg/admin/migration/util.go
index 444e852405..e3022a2847 100644
--- a/pkg/admin/migration/util.go
+++ b/pkg/admin/migration/util.go
@@ -6,18 +6,9 @@ import (
 	"strings"
 
 	"k8s.io/apimachinery/pkg/api/errors"
-	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
-	apitypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/net"
 )
 
-func getNamespacedName(item *unstructured.Unstructured) apitypes.NamespacedName {
-	return apitypes.NamespacedName{
-		Name:      item.GetName(),
-		Namespace: item.GetNamespace(),
-	}
-}
-
 func canRetry(err error) bool {
 	err = interpret(err)
 	if temp, ok := err.(TemporaryError); ok && !temp.Temporary() {
diff --git a/pkg/admin/prerun/prerun.go b/pkg/admin/prerun/prerun.go
index e5cf64e5f3..6b5e8df409 100644
--- a/pkg/admin/prerun/prerun.go
+++ b/pkg/admin/prerun/prerun.go
@@ -214,7 +214,6 @@ func (pr *PreRun) regularPrerun() error {
 }
 
 func (pr *PreRun) migrate() error {
-
 	// Run microshift in minimal mode
 	stop, err := runMinimalMicroshift(pr.config)
 	if err != nil {