USHIFT-1083 Backup data if system was healthy

packaging/greenboot/microshift_set_healthy.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -ex
+
+HEALTH=healthy
+
+SCRIPT_NAME=$(basename "$0")
+if [ "$(id -u)" -ne 0 ] ; then
+    echo "The '${SCRIPT_NAME}' script must be run with the 'root' user privileges"
+    exit 1
+fi
+
+if [ ! -f /run/ostree-booted ]; then
+    echo "System is not booted with ostree"
+    exit 0
+fi
+
+mkdir -p /var/lib/microshift-backups
+
+boot=$(tr -d '-' < /proc/sys/kernel/random/boot_id)
+deploy=$(rpm-ostree status --booted --jsonpath='$.deployments[0].id' | jq -r '.[0]')
+jq \
+    --null-input \
+    --arg health "${HEALTH}" \
+    --arg deploy "${deploy}" \
+    --arg boot "${boot}" \
+    '{ "health": $health, "deployment_id": $deploy, "boot_id": $boot }' > /var/lib/microshift-backups/health.json

packaging/greenboot/microshift_set_unhealthy.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+set -ex
+
+HEALTH=unhealthy
+
+SCRIPT_NAME=$(basename "$0")
+if [ "$(id -u)" -ne 0 ] ; then
+    echo "The '${SCRIPT_NAME}' script must be run with the 'root' user privileges"
+    exit 1
+fi
+
+if [ ! -f /run/ostree-booted ]; then
+    echo "System is not booted with ostree"
+    exit 0
+fi
+
+mkdir -p /var/lib/microshift-backups
+
+boot=$(tr -d '-' < /proc/sys/kernel/random/boot_id)
+deploy=$(rpm-ostree status --booted --jsonpath='$.deployments[0].id' | jq -r '.[0]')
+jq \
+    --null-input \
+    --arg health "${HEALTH}" \
+    --arg deploy "${deploy}" \
+    --arg boot "${boot}" \
+    '{ "health": $health, "deployment_id": $deploy, "boot_id": $boot }' > /var/lib/microshift-backups/health.json

packaging/rpm/microshift.spec

@@ -151,6 +151,9 @@ install -p -m755 scripts/microshift-cleanup-data.sh %{buildroot}%{_bindir}/micro
 restorecon -v %{buildroot}%{_bindir}/microshift
 restorecon -v %{buildroot}%{_bindir}/microshift-etcd
 
+install -d -m755 %{buildroot}{_sharedstatedir}/microshift
+install -d -m755 %{buildroot}{_sharedstatedir}/microshift-backups
+
 install -d -m755 %{buildroot}%{_sysconfdir}/crio/crio.conf.d
 
 %ifarch %{arm} aarch64
@@ -208,12 +211,18 @@ install -d %{buildroot}%{_datadir}/selinux/packages/%{selinuxtype}
 install -m644 packaging/selinux/microshift.pp.bz2 %{buildroot}%{_datadir}/selinux/packages/%{selinuxtype}
 
 # Greenboot scripts
+install -d -m755 %{buildroot}%{_datadir}/microshift/functions
+install -p -m644 packaging/greenboot/functions.sh %{buildroot}%{_datadir}/microshift/functions/greenboot.sh
+
 install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/check/required.d
-install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/red.d
 install -p -m755 packaging/greenboot/microshift-running-check.sh %{buildroot}%{_sysconfdir}/greenboot/check/required.d/40_microshift_running_check.sh
+
+install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/red.d
 install -p -m755 packaging/greenboot/microshift-pre-rollback.sh %{buildroot}%{_sysconfdir}/greenboot/red.d/40_microshift_pre_rollback.sh
-install -d -m755 %{buildroot}%{_datadir}/microshift/functions
-install -p -m644 packaging/greenboot/functions.sh %{buildroot}%{_datadir}/microshift/functions/greenboot.sh
+install -p -m755 packaging/greenboot/microshift_set_unhealthy.sh %{buildroot}%{_sysconfdir}/greenboot/red.d/40_microshift_set_unhealthy.sh
+
+install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/green.d
+install -p -m755 packaging/greenboot/microshift_set_healthy.sh %{buildroot}%{_sysconfdir}/greenboot/green.d/40_microshift_set_healthy.sh
 
 %post
 
@@ -295,6 +304,8 @@ systemctl enable --now --quiet openvswitch || true
 %files greenboot
 %{_sysconfdir}/greenboot/check/required.d/40_microshift_running_check.sh
 %{_sysconfdir}/greenboot/red.d/40_microshift_pre_rollback.sh
+%{_sysconfdir}/greenboot/red.d/40_microshift_set_unhealthy.sh
+%{_sysconfdir}/greenboot/green.d/40_microshift_set_healthy.sh
 %{_datadir}/microshift/functions/greenboot.sh
 
 # Use Git command to generate the log and replace the VERSION string

pkg/admin/data/data_manager.go

@@ -3,6 +3,7 @@ package data
 import (
 	"bytes"
 	"fmt"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
@@ -42,6 +43,26 @@ func (dm *manager) BackupExists(name BackupName) (bool, error) {
 	return pathExists(dm.GetBackupPath(name))
 }
 
+func (dm *manager) RemoveBackup(name BackupName) error {
+	return os.RemoveAll(dm.GetBackupPath(name))
+}
+
+func (dm *manager) GetBackupList() ([]BackupName, error) {
+	files, err := os.ReadDir(config.BackupsDir)
+	if err != nil {
+		return nil, err
+	}
+
+	backups := make([]BackupName, 0, len(files))
+	for _, file := range files {
+		if file.IsDir() {
+			backups = append(backups, BackupName(file.Name()))
+		}
+	}
+
+	return backups, nil
+}
+
 func (dm *manager) Backup(name BackupName) error {
 	klog.InfoS("Backing up the data",
 		"storage", dm.storage, "name", name, "data", config.DataDir)
@@ -50,6 +71,13 @@ func (dm *manager) Backup(name BackupName) error {
 		return &EmptyArgErr{"name"}
 	}
 
+	if exists, err := dm.BackupExists(name); err != nil {
+		return fmt.Errorf("checking if backup %s exists failed: %w", name, err)
+	} else if exists {
+		klog.ErrorS(nil, "Backup already exists - name should be unique", "name", name)
+		return fmt.Errorf("backup %s already exists", name)
+	}
+
 	if found, err := pathExists(string(dm.storage)); err != nil {
 		return err
 	} else if !found {

pkg/admin/data/types.go

@@ -21,4 +21,6 @@ type Manager interface {
 
 	BackupExists(BackupName) (bool, error)
 	GetBackupPath(BackupName) string
+	GetBackupList() ([]BackupName, error)
+	RemoveBackup(BackupName) error
 }

pkg/admin/prerun/prerun.go

@@ -0,0 +1,150 @@
+package prerun
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/openshift/microshift/pkg/admin/data"
+	"github.com/openshift/microshift/pkg/config"
+	"github.com/openshift/microshift/pkg/util"
+	"k8s.io/klog/v2"
+)
+
+var (
+	errHealthFileDoesNotExist = errors.New("health file does not exist")
+)
+
+type HealthInfo struct {
+	Health       string `json:"health"`
+	DeploymentID string `json:"deployment_id"`
+	BootID       string `json:"boot_id"`
+}
+
+func (hi *HealthInfo) BackupName() data.BackupName {
+	return data.BackupName(fmt.Sprintf("%s_%s", hi.DeploymentID, hi.BootID))
+}
+
+func (hi *HealthInfo) IsHealthy() bool {
+	return hi.Health == "healthy"
+}
+
+func Perform() error {
+	health, err := getHealthInfo()
+	if err != nil {
+		if errors.Is(err, errHealthFileDoesNotExist) {
+			klog.InfoS("Health file does not exist - skipping backup")
+			return nil
+		}
+		klog.ErrorS(err, "Failed to load health from disk")
+		return err
+	}
+	klog.InfoS("Loaded health info from the disk", "health", health)
+
+	if isCurr, err := containsCurrentBootID(health.BootID); err != nil {
+		return err
+	} else if isCurr {
+		klog.InfoS("Health file contains current boot - skipping backup")
+		return nil
+	}
+
+	if !health.IsHealthy() {
+		klog.InfoS("System was not healthy - skipping backup")
+		return nil
+	}
+
+	dataManager, err := data.NewManager(config.BackupsDir)
+	if err != nil {
+		return err
+	}
+
+	existingBackups, err := dataManager.GetBackupList()
+	if err != nil {
+		return err
+	}
+
+	// get list of already existing backups for deployment ID persisted in health file
+	// after creating backup, the list will be used to remove older backups
+	// (so only the most recent one for specific deployment is kept)
+	backupsForDeployment := getExistingBackupsForTheDeployment(existingBackups, health.DeploymentID)
+
+	newBackupName := health.BackupName()
+	if backupAlreadyExists(backupsForDeployment, newBackupName) {
+		klog.InfoS("Backup already exists", "name", newBackupName)
+		return nil
+	}
+
+	if err := dataManager.Backup(newBackupName); err != nil {
+		return err
+	}
+
+	removeOldBackups(dataManager, backupsForDeployment)
+
+	return nil
+}
+
+func containsCurrentBootID(id string) (bool, error) {
+	path := "/proc/sys/kernel/random/boot_id"
+	content, err := os.ReadFile(path)
+	if err != nil {
+		klog.ErrorS(err, "Failed to read file", "path", path)
+		return false, fmt.Errorf("reading file %s failed: %w", path, err)
+	}
+	currentBootID := strings.ReplaceAll(strings.TrimSpace(string(content)), "-", "")
+	klog.InfoS("Comparing boot IDs", "current", currentBootID, "toCompare", id)
+	return id == currentBootID, nil
+}
+
+func getHealthInfo() (*HealthInfo, error) {
+	path := "/var/lib/microshift-backups/health.json"
+	if exists, err := util.PathExists(path); err != nil {
+		return nil, err
+	} else if !exists {
+		return nil, errHealthFileDoesNotExist
+	}
+
+	content, err := os.ReadFile(path)
+	if err != nil {
+		klog.ErrorS(err, "Failed to read file", "path", path)
+		return nil, err
+	}
+
+	health := &HealthInfo{}
+	if err := json.Unmarshal(content, &health); err != nil {
+		klog.ErrorS(err, "Failed to unmarshal file to json", "content", string(content))
+		return nil, err
+	}
+	return health, nil
+}
+
+func getExistingBackupsForTheDeployment(existingBackups []data.BackupName, deployID string) []data.BackupName {
+	existingDeploymentBackups := make([]data.BackupName, 0)
+
+	for _, existingBackup := range existingBackups {
+		if strings.HasPrefix(string(existingBackup), deployID) {
+			existingDeploymentBackups = append(existingDeploymentBackups, existingBackup)
+		}
+	}
+
+	return existingDeploymentBackups
+}
+
+func backupAlreadyExists(existingBackups []data.BackupName, name data.BackupName) bool {
+	for _, backup := range existingBackups {
+		if backup == name {
+			return true
+		}
+	}
+	return false
+}
+
+func removeOldBackups(dataManager data.Manager, backups []data.BackupName) {
+	for _, b := range backups {
+		klog.InfoS("Removing older backup", "name", b)
+		if err := dataManager.RemoveBackup(b); err != nil {
+			klog.ErrorS(err, "Failed to remove backup", "name", b)
+		}
+	}
+}

pkg/cmd/run.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/coreos/go-systemd/daemon"
+	"github.com/openshift/microshift/pkg/admin/prerun"
 	"github.com/openshift/microshift/pkg/config"
 	"github.com/openshift/microshift/pkg/controllers"
 	"github.com/openshift/microshift/pkg/kustomize"
@@ -82,6 +83,10 @@ func RunMicroshift(cfg *config.Config) error {
 		klog.Fatalf("MicroShift must be run privileged")
 	}
 
+	if err := prerun.Perform(); err != nil {
+		return err
+	}
+
 	logConfig(cfg)
 
 	// TO-DO: When multi-node is ready, we need to add the controller host-name/mDNS hostname

pkg/util/util.go

@@ -25,7 +25,7 @@ func PathExists(path string) (bool, error) {
 	} else if errors.Is(err, os.ErrNotExist) {
 		return false, nil
 	} else {
-		return false, err
+		return false, fmt.Errorf("checking if path (%s) exists failed: %w", path, err)
 	}
 }