diff --git a/packaging/greenboot/microshift_set_healthy.sh b/packaging/greenboot/microshift_set_healthy.sh new file mode 100644 index 0000000000..b7fb3cde6c --- /dev/null +++ b/packaging/greenboot/microshift_set_healthy.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -ex + +HEALTH=healthy + +SCRIPT_NAME=$(basename "$0") +if [ "$(id -u)" -ne 0 ] ; then + echo "The '${SCRIPT_NAME}' script must be run with the 'root' user privileges" + exit 1 +fi + +if [ ! -f /run/ostree-booted ]; then + echo "System is not booted with ostree" + exit 0 +fi + +mkdir -p /var/lib/microshift-backups + +boot=$(tr -d '-' < /proc/sys/kernel/random/boot_id) +deploy=$(rpm-ostree status --booted --jsonpath='$.deployments[0].id' | jq -r '.[0]') +jq \ + --null-input \ + --arg health "${HEALTH}" \ + --arg deploy "${deploy}" \ + --arg boot "${boot}" \ + '{ "health": $health, "deployment_id": $deploy, "boot_id": $boot }' > /var/lib/microshift-backups/health.json diff --git a/packaging/greenboot/microshift_set_unhealthy.sh b/packaging/greenboot/microshift_set_unhealthy.sh new file mode 100644 index 0000000000..6ceab0d4b7 --- /dev/null +++ b/packaging/greenboot/microshift_set_unhealthy.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -ex + +HEALTH=unhealthy + +SCRIPT_NAME=$(basename "$0") +if [ "$(id -u)" -ne 0 ] ; then + echo "The '${SCRIPT_NAME}' script must be run with the 'root' user privileges" + exit 1 +fi + +if [ ! -f /run/ostree-booted ]; then + echo "System is not booted with ostree" + exit 0 +fi + +mkdir -p /var/lib/microshift-backups + +boot=$(tr -d '-' < /proc/sys/kernel/random/boot_id) +deploy=$(rpm-ostree status --booted --jsonpath='$.deployments[0].id' | jq -r '.[0]') +jq \ + --null-input \ + --arg health "${HEALTH}" \ + --arg deploy "${deploy}" \ + --arg boot "${boot}" \ + '{ "health": $health, "deployment_id": $deploy, "boot_id": $boot }' > /var/lib/microshift-backups/health.json diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index dea9190d83..d571942344 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -151,6 +151,9 @@ install -p -m755 scripts/microshift-cleanup-data.sh %{buildroot}%{_bindir}/micro restorecon -v %{buildroot}%{_bindir}/microshift restorecon -v %{buildroot}%{_bindir}/microshift-etcd +install -d -m755 %{buildroot}{_sharedstatedir}/microshift +install -d -m755 %{buildroot}{_sharedstatedir}/microshift-backups + install -d -m755 %{buildroot}%{_sysconfdir}/crio/crio.conf.d %ifarch %{arm} aarch64 @@ -208,12 +211,18 @@ install -d %{buildroot}%{_datadir}/selinux/packages/%{selinuxtype} install -m644 packaging/selinux/microshift.pp.bz2 %{buildroot}%{_datadir}/selinux/packages/%{selinuxtype} # Greenboot scripts +install -d -m755 %{buildroot}%{_datadir}/microshift/functions +install -p -m644 packaging/greenboot/functions.sh %{buildroot}%{_datadir}/microshift/functions/greenboot.sh + install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/check/required.d -install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/red.d install -p -m755 packaging/greenboot/microshift-running-check.sh %{buildroot}%{_sysconfdir}/greenboot/check/required.d/40_microshift_running_check.sh + +install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/red.d install -p -m755 packaging/greenboot/microshift-pre-rollback.sh %{buildroot}%{_sysconfdir}/greenboot/red.d/40_microshift_pre_rollback.sh -install -d -m755 %{buildroot}%{_datadir}/microshift/functions -install -p -m644 packaging/greenboot/functions.sh %{buildroot}%{_datadir}/microshift/functions/greenboot.sh +install -p -m755 packaging/greenboot/microshift_set_unhealthy.sh %{buildroot}%{_sysconfdir}/greenboot/red.d/40_microshift_set_unhealthy.sh + +install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/green.d +install -p -m755 packaging/greenboot/microshift_set_healthy.sh %{buildroot}%{_sysconfdir}/greenboot/green.d/40_microshift_set_healthy.sh %post @@ -295,6 +304,8 @@ systemctl enable --now --quiet openvswitch || true %files greenboot %{_sysconfdir}/greenboot/check/required.d/40_microshift_running_check.sh %{_sysconfdir}/greenboot/red.d/40_microshift_pre_rollback.sh +%{_sysconfdir}/greenboot/red.d/40_microshift_set_unhealthy.sh +%{_sysconfdir}/greenboot/green.d/40_microshift_set_healthy.sh %{_datadir}/microshift/functions/greenboot.sh # Use Git command to generate the log and replace the VERSION string diff --git a/pkg/admin/data/data_manager.go b/pkg/admin/data/data_manager.go index 64fcbf4605..a2e8318f18 100644 --- a/pkg/admin/data/data_manager.go +++ b/pkg/admin/data/data_manager.go @@ -3,6 +3,7 @@ package data import ( "bytes" "fmt" + "os" "os/exec" "path/filepath" "strings" @@ -42,6 +43,26 @@ func (dm *manager) BackupExists(name BackupName) (bool, error) { return pathExists(dm.GetBackupPath(name)) } +func (dm *manager) RemoveBackup(name BackupName) error { + return os.RemoveAll(dm.GetBackupPath(name)) +} + +func (dm *manager) GetBackupList() ([]BackupName, error) { + files, err := os.ReadDir(config.BackupsDir) + if err != nil { + return nil, err + } + + backups := make([]BackupName, 0, len(files)) + for _, file := range files { + if file.IsDir() { + backups = append(backups, BackupName(file.Name())) + } + } + + return backups, nil +} + func (dm *manager) Backup(name BackupName) error { klog.InfoS("Backing up the data", "storage", dm.storage, "name", name, "data", config.DataDir) @@ -50,6 +71,13 @@ func (dm *manager) Backup(name BackupName) error { return &EmptyArgErr{"name"} } + if exists, err := dm.BackupExists(name); err != nil { + return fmt.Errorf("checking if backup %s exists failed: %w", name, err) + } else if exists { + klog.ErrorS(nil, "Backup already exists - name should be unique", "name", name) + return fmt.Errorf("backup %s already exists", name) + } + if found, err := pathExists(string(dm.storage)); err != nil { return err } else if !found { diff --git a/pkg/admin/data/types.go b/pkg/admin/data/types.go index d11d59d104..ecabc3df53 100644 --- a/pkg/admin/data/types.go +++ b/pkg/admin/data/types.go @@ -21,4 +21,6 @@ type Manager interface { BackupExists(BackupName) (bool, error) GetBackupPath(BackupName) string + GetBackupList() ([]BackupName, error) + RemoveBackup(BackupName) error } diff --git a/pkg/admin/prerun/prerun.go b/pkg/admin/prerun/prerun.go new file mode 100644 index 0000000000..59e3da61c4 --- /dev/null +++ b/pkg/admin/prerun/prerun.go @@ -0,0 +1,150 @@ +package prerun + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "strings" + + "github.com/openshift/microshift/pkg/admin/data" + "github.com/openshift/microshift/pkg/config" + "github.com/openshift/microshift/pkg/util" + "k8s.io/klog/v2" +) + +var ( + errHealthFileDoesNotExist = errors.New("health file does not exist") +) + +type HealthInfo struct { + Health string `json:"health"` + DeploymentID string `json:"deployment_id"` + BootID string `json:"boot_id"` +} + +func (hi *HealthInfo) BackupName() data.BackupName { + return data.BackupName(fmt.Sprintf("%s_%s", hi.DeploymentID, hi.BootID)) +} + +func (hi *HealthInfo) IsHealthy() bool { + return hi.Health == "healthy" +} + +func Perform() error { + health, err := getHealthInfo() + if err != nil { + if errors.Is(err, errHealthFileDoesNotExist) { + klog.InfoS("Health file does not exist - skipping backup") + return nil + } + klog.ErrorS(err, "Failed to load health from disk") + return err + } + klog.InfoS("Loaded health info from the disk", "health", health) + + if isCurr, err := containsCurrentBootID(health.BootID); err != nil { + return err + } else if isCurr { + klog.InfoS("Health file contains current boot - skipping backup") + return nil + } + + if !health.IsHealthy() { + klog.InfoS("System was not healthy - skipping backup") + return nil + } + + dataManager, err := data.NewManager(config.BackupsDir) + if err != nil { + return err + } + + existingBackups, err := dataManager.GetBackupList() + if err != nil { + return err + } + + // get list of already existing backups for deployment ID persisted in health file + // after creating backup, the list will be used to remove older backups + // (so only the most recent one for specific deployment is kept) + backupsForDeployment := getExistingBackupsForTheDeployment(existingBackups, health.DeploymentID) + + newBackupName := health.BackupName() + if backupAlreadyExists(backupsForDeployment, newBackupName) { + klog.InfoS("Backup already exists", "name", newBackupName) + return nil + } + + if err := dataManager.Backup(newBackupName); err != nil { + return err + } + + removeOldBackups(dataManager, backupsForDeployment) + + return nil +} + +func containsCurrentBootID(id string) (bool, error) { + path := "/proc/sys/kernel/random/boot_id" + content, err := os.ReadFile(path) + if err != nil { + klog.ErrorS(err, "Failed to read file", "path", path) + return false, fmt.Errorf("reading file %s failed: %w", path, err) + } + currentBootID := strings.ReplaceAll(strings.TrimSpace(string(content)), "-", "") + klog.InfoS("Comparing boot IDs", "current", currentBootID, "toCompare", id) + return id == currentBootID, nil +} + +func getHealthInfo() (*HealthInfo, error) { + path := "/var/lib/microshift-backups/health.json" + if exists, err := util.PathExists(path); err != nil { + return nil, err + } else if !exists { + return nil, errHealthFileDoesNotExist + } + + content, err := os.ReadFile(path) + if err != nil { + klog.ErrorS(err, "Failed to read file", "path", path) + return nil, err + } + + health := &HealthInfo{} + if err := json.Unmarshal(content, &health); err != nil { + klog.ErrorS(err, "Failed to unmarshal file to json", "content", string(content)) + return nil, err + } + return health, nil +} + +func getExistingBackupsForTheDeployment(existingBackups []data.BackupName, deployID string) []data.BackupName { + existingDeploymentBackups := make([]data.BackupName, 0) + + for _, existingBackup := range existingBackups { + if strings.HasPrefix(string(existingBackup), deployID) { + existingDeploymentBackups = append(existingDeploymentBackups, existingBackup) + } + } + + return existingDeploymentBackups +} + +func backupAlreadyExists(existingBackups []data.BackupName, name data.BackupName) bool { + for _, backup := range existingBackups { + if backup == name { + return true + } + } + return false +} + +func removeOldBackups(dataManager data.Manager, backups []data.BackupName) { + for _, b := range backups { + klog.InfoS("Removing older backup", "name", b) + if err := dataManager.RemoveBackup(b); err != nil { + klog.ErrorS(err, "Failed to remove backup", "name", b) + } + } +} diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 71e0946a46..dab2d93d1f 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -10,6 +10,7 @@ import ( "time" "github.com/coreos/go-systemd/daemon" + "github.com/openshift/microshift/pkg/admin/prerun" "github.com/openshift/microshift/pkg/config" "github.com/openshift/microshift/pkg/controllers" "github.com/openshift/microshift/pkg/kustomize" @@ -82,6 +83,10 @@ func RunMicroshift(cfg *config.Config) error { klog.Fatalf("MicroShift must be run privileged") } + if err := prerun.Perform(); err != nil { + return err + } + logConfig(cfg) // TO-DO: When multi-node is ready, we need to add the controller host-name/mDNS hostname diff --git a/pkg/util/util.go b/pkg/util/util.go index 61725b28d2..e7fb5d787d 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -25,7 +25,7 @@ func PathExists(path string) (bool, error) { } else if errors.Is(err, os.ErrNotExist) { return false, nil } else { - return false, err + return false, fmt.Errorf("checking if path (%s) exists failed: %w", path, err) } }