From aab5aada2931afdde586e0a09a8f1784acb6f216 Mon Sep 17 00:00:00 2001 From: Deepak Raj D S Date: Wed, 6 Apr 2022 13:07:39 -0400 Subject: [PATCH 1/4] Adding function to verify BackUp / Restore data --- tests/e2e/lib/apps.go | 57 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 6f4a1e6fef9..88e3c4c0494 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -1,14 +1,19 @@ package lib import ( + "bytes" "context" "errors" "fmt" + "io/ioutil" + "log" + "net/http" "os" "os/exec" "github.com/onsi/ginkgo/v2" ocpappsv1 "github.com/openshift/api/apps/v1" + routev1 "github.com/openshift/api/route/v1" security "github.com/openshift/api/security/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -18,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/serializer/yaml" "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" ) func InstallApplication(ocClient client.Client, file string) error { @@ -197,3 +203,54 @@ func RunMustGather(oc_cli string, artifact_dir string) error { _, err := cmd.Output() return err } + +func VerifyBackUpRestoreData(artifact_dir string, namespace string, routeName string, app string) error { + log.Printf("Verifying backup/restore data of %s", app) + appRoute := &routev1.Route{} + clientv1, err := client.New(config.GetConfigOrDie(), client.Options{}) + if err != nil { + return err + } + backupFile := artifact_dir + "/backup-data.txt" + routev1.AddToScheme(clientv1.Scheme()) + err = clientv1.Get(context.Background(), client.ObjectKey{ + Namespace: namespace, + Name: routeName, + }, appRoute) + if err != nil { + return err + } + appApi := "http://" + appRoute.Spec.Host + switch app { + case "todolist": + appApi += "/todo-completed" + case "parks-app": + appApi += "/parks" + } + resp, err := http.Get(appApi) + if err != nil { + return err + } + defer resp.Body.Close() + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + if _, err := os.Stat(backupFile); err == nil { + backupData, err := os.ReadFile(backupFile) + if err != nil { + return err + } + os.Remove(backupFile) + if bytes.Compare(backupData, respData) != 0 { + return errors.New("Backup and Restore Data are not the same") + } + } else if errors.Is(err, os.ErrNotExist) { + err := os.WriteFile(backupFile, respData, 0644) + if err != nil { + return err + } + } + return nil +} From 008773ea51025d130fbd0687aa496ba9cf7204e8 Mon Sep 17 00:00:00 2001 From: Deepak Raj D S Date: Wed, 20 Apr 2022 10:15:44 -0400 Subject: [PATCH 2/4] Adding verification for mysql --- tests/e2e/backup_restore_suite_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index b851f7be395..17cea5ea594 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -47,6 +47,7 @@ var _ = Describe("AWS backup restore tests", func() { parksAppReady := VerificationFunction(func(ocClient client.Client, namespace string) error { Eventually(IsDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) + // err := VerifyBackUpRestoreData(artifact_dir, namespace, "restify", "parks-app") // TODO: VERIFY PARKS APP DATA return nil }) mysqlReady := VerificationFunction(func(ocClient client.Client, namespace string) error { @@ -60,7 +61,8 @@ var _ = Describe("AWS backup restore tests", func() { if !exists { return errors.New("did not find MYSQL scc") } - return nil + err = VerifyBackUpRestoreData(artifact_dir, namespace, "todolist-route", "todolist") + return err }) DescribeTable("backup and restore applications", From 45917b03403f7b470a4d8b1315f98814a8d2dde7 Mon Sep 17 00:00:00 2001 From: Deepak Raj D S Date: Thu, 21 Apr 2022 12:36:48 -0400 Subject: [PATCH 3/4] Adding timeout / retry function --- tests/e2e/lib/apps.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 88e3c4c0494..06b44579d03 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "os/exec" + "time" "github.com/onsi/ginkgo/v2" ocpappsv1 "github.com/openshift/api/apps/v1" @@ -233,6 +234,22 @@ func VerifyBackUpRestoreData(artifact_dir string, namespace string, routeName st } defer resp.Body.Close() + if resp.StatusCode != 200 { // # TODO: NEED TO FIND A BETTER WAY TO DEBUG RESPONSE + var retrySchedule = []time.Duration{ + 15 * time.Second, + 1 * time.Minute, + 2 * time.Minute, + } + for _, backoff := range retrySchedule { + resp, err = http.Get(appApi) + if resp.StatusCode != 200 { + log.Printf("Request error: %+v\n", err) + log.Printf("Retrying in %v\n", backoff) + time.Sleep(backoff) + } + } + } + respData, err := ioutil.ReadAll(resp.Body) if err != nil { return err @@ -243,7 +260,7 @@ func VerifyBackUpRestoreData(artifact_dir string, namespace string, routeName st return err } os.Remove(backupFile) - if bytes.Compare(backupData, respData) != 0 { + if !bytes.Equal(backupData, respData) { return errors.New("Backup and Restore Data are not the same") } } else if errors.Is(err, os.ErrNotExist) { From 7d45f424eeb6ef7c642484fbc4c1b6bfd1ccc95a Mon Sep 17 00:00:00 2001 From: Deepak Raj D S Date: Fri, 29 Apr 2022 01:14:48 +0530 Subject: [PATCH 4/4] Changing VerifyBackupRestoreData fn name --- tests/e2e/backup_restore_suite_test.go | 4 ++-- tests/e2e/lib/apps.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/e2e/backup_restore_suite_test.go b/tests/e2e/backup_restore_suite_test.go index 17cea5ea594..81e0af1431d 100755 --- a/tests/e2e/backup_restore_suite_test.go +++ b/tests/e2e/backup_restore_suite_test.go @@ -47,7 +47,7 @@ var _ = Describe("AWS backup restore tests", func() { parksAppReady := VerificationFunction(func(ocClient client.Client, namespace string) error { Eventually(IsDCReady(ocClient, "parks-app", "restify"), timeoutMultiplier*time.Minute*10, time.Second*10).Should(BeTrue()) - // err := VerifyBackUpRestoreData(artifact_dir, namespace, "restify", "parks-app") // TODO: VERIFY PARKS APP DATA + // err := VerifyBackupRestoreData(artifact_dir, namespace, "restify", "parks-app") // TODO: VERIFY PARKS APP DATA return nil }) mysqlReady := VerificationFunction(func(ocClient client.Client, namespace string) error { @@ -61,7 +61,7 @@ var _ = Describe("AWS backup restore tests", func() { if !exists { return errors.New("did not find MYSQL scc") } - err = VerifyBackUpRestoreData(artifact_dir, namespace, "todolist-route", "todolist") + err = VerifyBackupRestoreData(artifact_dir, namespace, "todolist-route", "todolist") return err }) diff --git a/tests/e2e/lib/apps.go b/tests/e2e/lib/apps.go index 06b44579d03..9261db2b999 100755 --- a/tests/e2e/lib/apps.go +++ b/tests/e2e/lib/apps.go @@ -205,7 +205,7 @@ func RunMustGather(oc_cli string, artifact_dir string) error { return err } -func VerifyBackUpRestoreData(artifact_dir string, namespace string, routeName string, app string) error { +func VerifyBackupRestoreData(artifact_dir string, namespace string, routeName string, app string) error { log.Printf("Verifying backup/restore data of %s", app) appRoute := &routev1.Route{} clientv1, err := client.New(config.GetConfigOrDie(), client.Options{})