From 6f414713ed63216b75917c227cdb78b94b0a5a80 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Fri, 2 Jun 2023 13:16:23 +0200 Subject: [PATCH 1/7] simple backup on boot --- packaging/greenboot/green.sh | 19 +++++++++ packaging/greenboot/red.sh | 19 +++++++++ packaging/rpm/microshift.spec | 14 +++++-- pkg/admin/data/data_manager.go | 28 +++++++++++++ pkg/cmd/run.go | 74 ++++++++++++++++++++++++++++++++++ 5 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 packaging/greenboot/green.sh create mode 100644 packaging/greenboot/red.sh diff --git a/packaging/greenboot/green.sh b/packaging/greenboot/green.sh new file mode 100644 index 0000000000..7a112ba0a5 --- /dev/null +++ b/packaging/greenboot/green.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -ex + +HEALTH=healthy + +if [ ! -f /run/ostree-booted ]; then + echo "System is not booted with ostree" + exit 0 +fi + +mkdir -p /var/lib/microshift-backups + +ID=$(rpm-ostree status --booted --jsonpath='$.deployments[0].id' | jq -r '.[0]') +jq \ + --null-input \ + --arg health "${HEALTH}" \ + --arg id "${ID}" \ + '{ "health": $health, "deployment_id": $id }' > /var/lib/microshift-backups/health.json diff --git a/packaging/greenboot/red.sh b/packaging/greenboot/red.sh new file mode 100644 index 0000000000..3d382ee1e6 --- /dev/null +++ b/packaging/greenboot/red.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +set -ex + +HEALTH=unhealthy + +if [ ! -f /run/ostree-booted ]; then + echo "System is not booted with ostree" + exit 0 +fi + +mkdir -p /var/lib/microshift-backups + +ID=$(rpm-ostree status --booted --jsonpath='$.deployments[0].id' | jq -r '.[0]') +jq \ + --null-input \ + --arg health "${HEALTH}" \ + --arg id "${ID}" \ + '{ "health": $health, "deployment_id": $id }' > /var/lib/microshift-backups/health.json diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index dea9190d83..917bc601f0 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -208,12 +208,18 @@ install -d %{buildroot}%{_datadir}/selinux/packages/%{selinuxtype} install -m644 packaging/selinux/microshift.pp.bz2 %{buildroot}%{_datadir}/selinux/packages/%{selinuxtype} # Greenboot scripts +install -d -m755 %{buildroot}%{_datadir}/microshift/functions +install -p -m644 packaging/greenboot/functions.sh %{buildroot}%{_datadir}/microshift/functions/greenboot.sh + install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/check/required.d -install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/red.d install -p -m755 packaging/greenboot/microshift-running-check.sh %{buildroot}%{_sysconfdir}/greenboot/check/required.d/40_microshift_running_check.sh + +install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/red.d install -p -m755 packaging/greenboot/microshift-pre-rollback.sh %{buildroot}%{_sysconfdir}/greenboot/red.d/40_microshift_pre_rollback.sh -install -d -m755 %{buildroot}%{_datadir}/microshift/functions -install -p -m644 packaging/greenboot/functions.sh %{buildroot}%{_datadir}/microshift/functions/greenboot.sh +install -p -m755 packaging/greenboot/red.sh %{buildroot}%{_sysconfdir}/greenboot/red.d/40_microshift_set_health.sh + +install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/green.d +install -p -m755 packaging/greenboot/green.sh %{buildroot}%{_sysconfdir}/greenboot/green.d/40_microshift_set_health.sh %post @@ -295,6 +301,8 @@ systemctl enable --now --quiet openvswitch || true %files greenboot %{_sysconfdir}/greenboot/check/required.d/40_microshift_running_check.sh %{_sysconfdir}/greenboot/red.d/40_microshift_pre_rollback.sh +%{_sysconfdir}/greenboot/red.d/40_microshift_set_health.sh +%{_sysconfdir}/greenboot/green.d/40_microshift_set_health.sh %{_datadir}/microshift/functions/greenboot.sh # Use Git command to generate the log and replace the VERSION string diff --git a/pkg/admin/data/data_manager.go b/pkg/admin/data/data_manager.go index 64fcbf4605..a2e8318f18 100644 --- a/pkg/admin/data/data_manager.go +++ b/pkg/admin/data/data_manager.go @@ -3,6 +3,7 @@ package data import ( "bytes" "fmt" + "os" "os/exec" "path/filepath" "strings" @@ -42,6 +43,26 @@ func (dm *manager) BackupExists(name BackupName) (bool, error) { return pathExists(dm.GetBackupPath(name)) } +func (dm *manager) RemoveBackup(name BackupName) error { + return os.RemoveAll(dm.GetBackupPath(name)) +} + +func (dm *manager) GetBackupList() ([]BackupName, error) { + files, err := os.ReadDir(config.BackupsDir) + if err != nil { + return nil, err + } + + backups := make([]BackupName, 0, len(files)) + for _, file := range files { + if file.IsDir() { + backups = append(backups, BackupName(file.Name())) + } + } + + return backups, nil +} + func (dm *manager) Backup(name BackupName) error { klog.InfoS("Backing up the data", "storage", dm.storage, "name", name, "data", config.DataDir) @@ -50,6 +71,13 @@ func (dm *manager) Backup(name BackupName) error { return &EmptyArgErr{"name"} } + if exists, err := dm.BackupExists(name); err != nil { + return fmt.Errorf("checking if backup %s exists failed: %w", name, err) + } else if exists { + klog.ErrorS(nil, "Backup already exists - name should be unique", "name", name) + return fmt.Errorf("backup %s already exists", name) + } + if found, err := pathExists(string(dm.storage)); err != nil { return err } else if !found { diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 71e0946a46..7c1b07bf1c 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "encoding/json" "fmt" "os" "os/signal" @@ -10,6 +11,7 @@ import ( "time" "github.com/coreos/go-systemd/daemon" + "github.com/openshift/microshift/pkg/admin/data" "github.com/openshift/microshift/pkg/config" "github.com/openshift/microshift/pkg/controllers" "github.com/openshift/microshift/pkg/kustomize" @@ -76,12 +78,84 @@ func logConfig(cfg *config.Config) { } } +func preRun() error { + healthFile := "/var/lib/microshift-backups/health.json" + exists, err := util.PathExists(healthFile) + if err != nil { + klog.ErrorS(err, "Could not access health file", "path", healthFile) + return err + } + if !exists { + klog.InfoS("Boot health information is missing - skipping backup") + return nil + } + + info := struct { + Health string `json:"health"` + DeploymentID string `json:"deployment_id"` + }{} + d, err := os.ReadFile(healthFile) + if err != nil { + klog.ErrorS(err, "Failed to read file", "path", healthFile) + return err + } + if err := json.Unmarshal(d, &info); err != nil { + klog.ErrorS(err, "Failed to unmarshal json", "data", string(d)) + return err + } + + klog.InfoS("Read health info", "info", info) + if info.Health != "healthy" { + return nil + } + + dm, err := data.NewManager(config.BackupsDir) + if err != nil { + return err + } + + backups, err := dm.GetBackupList() + if err != nil { + return err + } + + existingDeploymentBackups := make([]data.BackupName, 0) + for _, b := range backups { + if strings.Contains(string(b), info.DeploymentID) { + existingDeploymentBackups = append(existingDeploymentBackups, b) + } + } + + name := fmt.Sprintf("%s_%s", info.DeploymentID, time.Now().UTC().Format("20060102_150405")) + if err := dm.Backup(data.BackupName(name)); err != nil { + return err + } + + if len(existingDeploymentBackups) > 0 { + klog.InfoS("Removing old deployment backups", + "deployment", info.DeploymentID, + "backups", existingDeploymentBackups) + + for _, b := range existingDeploymentBackups { + if err := dm.RemoveBackup(b); err != nil { + klog.ErrorS(err, "Failed to remove backup", "name", b) + } + } + } + + return nil +} + func RunMicroshift(cfg *config.Config) error { // fail early if we don't have enough privileges if os.Geteuid() > 0 { klog.Fatalf("MicroShift must be run privileged") } + if err := preRun(); err != nil { + return err + } + logConfig(cfg) // TO-DO: When multi-node is ready, we need to add the controller host-name/mDNS hostname From fe53949c75ced3c4dfe159a801827de37f8dacea Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Fri, 2 Jun 2023 18:41:29 +0200 Subject: [PATCH 2/7] move prerun to separate file --- pkg/admin/prerun/prerun.go | 82 ++++++++++++++++++++++++++++++++++++++ pkg/cmd/run.go | 73 +-------------------------------- 2 files changed, 84 insertions(+), 71 deletions(-) create mode 100644 pkg/admin/prerun/prerun.go diff --git a/pkg/admin/prerun/prerun.go b/pkg/admin/prerun/prerun.go new file mode 100644 index 0000000000..1d03f52ad9 --- /dev/null +++ b/pkg/admin/prerun/prerun.go @@ -0,0 +1,82 @@ +package prerun + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/openshift/microshift/pkg/admin/data" + "github.com/openshift/microshift/pkg/config" + "github.com/openshift/microshift/pkg/util" + "k8s.io/klog/v2" +) + +func Perform() error { + healthFile := "/var/lib/microshift-backups/health.json" + exists, err := util.PathExists(healthFile) + if err != nil { + klog.ErrorS(err, "Could not access health file", "path", healthFile) + return err + } + if !exists { + klog.InfoS("Boot health information is missing - skipping backup") + return nil + } + + info := struct { + Health string `json:"health"` + DeploymentID string `json:"deployment_id"` + }{} + d, err := os.ReadFile(healthFile) + if err != nil { + klog.ErrorS(err, "Failed to read file", "path", healthFile) + return err + } + if err := json.Unmarshal(d, &info); err != nil { + klog.ErrorS(err, "Failed to unmarshal json", "data", string(d)) + return err + } + + klog.InfoS("Read health info", "info", info) + if info.Health != "healthy" { + return nil + } + + dm, err := data.NewManager(config.BackupsDir) + if err != nil { + return err + } + + backups, err := dm.GetBackupList() + if err != nil { + return err + } + + existingDeploymentBackups := make([]data.BackupName, 0) + for _, b := range backups { + if strings.Contains(string(b), info.DeploymentID) { + existingDeploymentBackups = append(existingDeploymentBackups, b) + } + } + + name := fmt.Sprintf("%s_%s", info.DeploymentID, time.Now().UTC().Format("20060102_150405")) + if err := dm.Backup(data.BackupName(name)); err != nil { + return err + } + + if len(existingDeploymentBackups) > 0 { + klog.InfoS("Removing old deployment backups", + "deployment", info.DeploymentID, + "backups", existingDeploymentBackups) + + for _, b := range existingDeploymentBackups { + if err := dm.RemoveBackup(b); err != nil { + klog.ErrorS(err, "Failed to remove backup", "name", b) + } + } + } + + return nil +} diff --git a/pkg/cmd/run.go b/pkg/cmd/run.go index 7c1b07bf1c..dab2d93d1f 100644 --- a/pkg/cmd/run.go +++ b/pkg/cmd/run.go @@ -2,7 +2,6 @@ package cmd import ( "context" - "encoding/json" "fmt" "os" "os/signal" @@ -11,7 +10,7 @@ import ( "time" "github.com/coreos/go-systemd/daemon" - "github.com/openshift/microshift/pkg/admin/data" + "github.com/openshift/microshift/pkg/admin/prerun" "github.com/openshift/microshift/pkg/config" "github.com/openshift/microshift/pkg/controllers" "github.com/openshift/microshift/pkg/kustomize" @@ -78,81 +77,13 @@ func logConfig(cfg *config.Config) { } } -func preRun() error { - healthFile := "/var/lib/microshift-backups/health.json" - exists, err := util.PathExists(healthFile) - if err != nil { - klog.ErrorS(err, "Could not access health file", "path", healthFile) - return err - } - if !exists { - klog.InfoS("Boot health information is missing - skipping backup") - return nil - } - - info := struct { - Health string `json:"health"` - DeploymentID string `json:"deployment_id"` - }{} - d, err := os.ReadFile(healthFile) - if err != nil { - klog.ErrorS(err, "Failed to read file", "path", healthFile) - return err - } - if err := json.Unmarshal(d, &info); err != nil { - klog.ErrorS(err, "Failed to unmarshal json", "data", string(d)) - return err - } - - klog.InfoS("Read health info", "info", info) - if info.Health != "healthy" { - return nil - } - - dm, err := data.NewManager(config.BackupsDir) - if err != nil { - return err - } - - backups, err := dm.GetBackupList() - if err != nil { - return err - } - - existingDeploymentBackups := make([]data.BackupName, 0) - for _, b := range backups { - if strings.Contains(string(b), info.DeploymentID) { - existingDeploymentBackups = append(existingDeploymentBackups, b) - } - } - - name := fmt.Sprintf("%s_%s", info.DeploymentID, time.Now().UTC().Format("20060102_150405")) - if err := dm.Backup(data.BackupName(name)); err != nil { - return err - } - - if len(existingDeploymentBackups) > 0 { - klog.InfoS("Removing old deployment backups", - "deployment", info.DeploymentID, - "backups", existingDeploymentBackups) - - for _, b := range existingDeploymentBackups { - if err := dm.RemoveBackup(b); err != nil { - klog.ErrorS(err, "Failed to remove backup", "name", b) - } - } - } - - return nil -} - func RunMicroshift(cfg *config.Config) error { // fail early if we don't have enough privileges if os.Geteuid() > 0 { klog.Fatalf("MicroShift must be run privileged") } - if err := preRun(); err != nil { + if err := prerun.Perform(); err != nil { return err } From adc4a1c389ad8a6723c985d7b519d6860caeb3ba Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Fri, 2 Jun 2023 19:02:07 +0200 Subject: [PATCH 3/7] use boot_id for backup name --- packaging/greenboot/green.sh | 8 +++++--- packaging/greenboot/red.sh | 8 +++++--- pkg/admin/prerun/prerun.go | 31 ++++++++++++++++++++++++++++--- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/packaging/greenboot/green.sh b/packaging/greenboot/green.sh index 7a112ba0a5..64961b2f6c 100644 --- a/packaging/greenboot/green.sh +++ b/packaging/greenboot/green.sh @@ -11,9 +11,11 @@ fi mkdir -p /var/lib/microshift-backups -ID=$(rpm-ostree status --booted --jsonpath='$.deployments[0].id' | jq -r '.[0]') +boot=$(tr -d '-' < /proc/sys/kernel/random/boot_id) +deploy=$(rpm-ostree status --booted --jsonpath='$.deployments[0].id' | jq -r '.[0]') jq \ --null-input \ --arg health "${HEALTH}" \ - --arg id "${ID}" \ - '{ "health": $health, "deployment_id": $id }' > /var/lib/microshift-backups/health.json + --arg deploy "${deploy}" \ + --arg boot "${boot}" \ + '{ "health": $health, "deployment_id": $deploy, "boot_id": $boot }' > /var/lib/microshift-backups/health.json diff --git a/packaging/greenboot/red.sh b/packaging/greenboot/red.sh index 3d382ee1e6..f99520d88d 100644 --- a/packaging/greenboot/red.sh +++ b/packaging/greenboot/red.sh @@ -11,9 +11,11 @@ fi mkdir -p /var/lib/microshift-backups -ID=$(rpm-ostree status --booted --jsonpath='$.deployments[0].id' | jq -r '.[0]') +boot=$(tr -d '-' < /proc/sys/kernel/random/boot_id) +deploy=$(rpm-ostree status --booted --jsonpath='$.deployments[0].id' | jq -r '.[0]') jq \ --null-input \ --arg health "${HEALTH}" \ - --arg id "${ID}" \ - '{ "health": $health, "deployment_id": $id }' > /var/lib/microshift-backups/health.json + --arg deploy "${deploy}" \ + --arg boot "${boot}" \ + '{ "health": $health, "deployment_id": $deploy, "boot_id": $boot }' > /var/lib/microshift-backups/health.json diff --git a/pkg/admin/prerun/prerun.go b/pkg/admin/prerun/prerun.go index 1d03f52ad9..cd170b2ffd 100644 --- a/pkg/admin/prerun/prerun.go +++ b/pkg/admin/prerun/prerun.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "strings" - "time" "github.com/openshift/microshift/pkg/admin/data" "github.com/openshift/microshift/pkg/config" @@ -25,9 +24,17 @@ func Perform() error { return nil } + currentBootID, err := getCurrentBootID() + if err != nil { + klog.ErrorS(err, "Failed to get current boot ID") + return err + } + klog.InfoS("Current boot", "id", currentBootID) + info := struct { Health string `json:"health"` DeploymentID string `json:"deployment_id"` + BootID string `json:"boot_id"` }{} d, err := os.ReadFile(healthFile) if err != nil { @@ -40,9 +47,16 @@ func Perform() error { } klog.InfoS("Read health info", "info", info) + + if info.BootID == currentBootID { + klog.InfoS("Current boot in health file - skipping backup") + return nil + } + if info.Health != "healthy" { return nil } + name := data.BackupName(fmt.Sprintf("%s_%s", info.DeploymentID, info.BootID)) dm, err := data.NewManager(config.BackupsDir) if err != nil { @@ -56,13 +70,16 @@ func Perform() error { existingDeploymentBackups := make([]data.BackupName, 0) for _, b := range backups { + if name == b { + klog.InfoS("Backup already exists", "deployment", info.DeploymentID, "boot", info.BootID) + return nil + } if strings.Contains(string(b), info.DeploymentID) { existingDeploymentBackups = append(existingDeploymentBackups, b) } } - name := fmt.Sprintf("%s_%s", info.DeploymentID, time.Now().UTC().Format("20060102_150405")) - if err := dm.Backup(data.BackupName(name)); err != nil { + if err := dm.Backup(name); err != nil { return err } @@ -80,3 +97,11 @@ func Perform() error { return nil } + +func getCurrentBootID() (string, error) { + content, err := os.ReadFile("/proc/sys/kernel/random/boot_id") + if err != nil { + return "", err + } + return strings.ReplaceAll(strings.TrimSpace(string(content)), "-", ""), nil +} From f8400fe30b79ac4fe6203d5334d3d19781c9bb37 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Mon, 5 Jun 2023 09:27:04 +0200 Subject: [PATCH 4/7] rename green red scripts --- .../greenboot/{green.sh => microshift_set_healthy.sh} | 0 .../greenboot/{red.sh => microshift_set_unhealthy.sh} | 0 packaging/rpm/microshift.spec | 8 ++++---- 3 files changed, 4 insertions(+), 4 deletions(-) rename packaging/greenboot/{green.sh => microshift_set_healthy.sh} (100%) rename packaging/greenboot/{red.sh => microshift_set_unhealthy.sh} (100%) diff --git a/packaging/greenboot/green.sh b/packaging/greenboot/microshift_set_healthy.sh similarity index 100% rename from packaging/greenboot/green.sh rename to packaging/greenboot/microshift_set_healthy.sh diff --git a/packaging/greenboot/red.sh b/packaging/greenboot/microshift_set_unhealthy.sh similarity index 100% rename from packaging/greenboot/red.sh rename to packaging/greenboot/microshift_set_unhealthy.sh diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index 917bc601f0..926fcc12f9 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -216,10 +216,10 @@ install -p -m755 packaging/greenboot/microshift-running-check.sh %{buildroot}%{_ install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/red.d install -p -m755 packaging/greenboot/microshift-pre-rollback.sh %{buildroot}%{_sysconfdir}/greenboot/red.d/40_microshift_pre_rollback.sh -install -p -m755 packaging/greenboot/red.sh %{buildroot}%{_sysconfdir}/greenboot/red.d/40_microshift_set_health.sh +install -p -m755 packaging/greenboot/microshift_set_unhealthy.sh %{buildroot}%{_sysconfdir}/greenboot/red.d/40_microshift_set_unhealthy.sh install -d -m755 %{buildroot}%{_sysconfdir}/greenboot/green.d -install -p -m755 packaging/greenboot/green.sh %{buildroot}%{_sysconfdir}/greenboot/green.d/40_microshift_set_health.sh +install -p -m755 packaging/greenboot/microshift_set_healthy.sh %{buildroot}%{_sysconfdir}/greenboot/green.d/40_microshift_set_healthy.sh %post @@ -301,8 +301,8 @@ systemctl enable --now --quiet openvswitch || true %files greenboot %{_sysconfdir}/greenboot/check/required.d/40_microshift_running_check.sh %{_sysconfdir}/greenboot/red.d/40_microshift_pre_rollback.sh -%{_sysconfdir}/greenboot/red.d/40_microshift_set_health.sh -%{_sysconfdir}/greenboot/green.d/40_microshift_set_health.sh +%{_sysconfdir}/greenboot/red.d/40_microshift_set_unhealthy.sh +%{_sysconfdir}/greenboot/green.d/40_microshift_set_healthy.sh %{_datadir}/microshift/functions/greenboot.sh # Use Git command to generate the log and replace the VERSION string From bd7639163c2970991dd2519f6c9a3ca5a55573e2 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Mon, 5 Jun 2023 09:31:59 +0200 Subject: [PATCH 5/7] green/red: check if root --- packaging/greenboot/microshift_set_healthy.sh | 6 ++++++ packaging/greenboot/microshift_set_unhealthy.sh | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/packaging/greenboot/microshift_set_healthy.sh b/packaging/greenboot/microshift_set_healthy.sh index 64961b2f6c..b7fb3cde6c 100644 --- a/packaging/greenboot/microshift_set_healthy.sh +++ b/packaging/greenboot/microshift_set_healthy.sh @@ -4,6 +4,12 @@ set -ex HEALTH=healthy +SCRIPT_NAME=$(basename "$0") +if [ "$(id -u)" -ne 0 ] ; then + echo "The '${SCRIPT_NAME}' script must be run with the 'root' user privileges" + exit 1 +fi + if [ ! -f /run/ostree-booted ]; then echo "System is not booted with ostree" exit 0 diff --git a/packaging/greenboot/microshift_set_unhealthy.sh b/packaging/greenboot/microshift_set_unhealthy.sh index f99520d88d..6ceab0d4b7 100644 --- a/packaging/greenboot/microshift_set_unhealthy.sh +++ b/packaging/greenboot/microshift_set_unhealthy.sh @@ -4,6 +4,12 @@ set -ex HEALTH=unhealthy +SCRIPT_NAME=$(basename "$0") +if [ "$(id -u)" -ne 0 ] ; then + echo "The '${SCRIPT_NAME}' script must be run with the 'root' user privileges" + exit 1 +fi + if [ ! -f /run/ostree-booted ]; then echo "System is not booted with ostree" exit 0 From f746089f1d6be817f0d5764db768189914a08be9 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Mon, 5 Jun 2023 09:54:45 +0200 Subject: [PATCH 6/7] add var-lib dirs to spec --- packaging/rpm/microshift.spec | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index 926fcc12f9..d571942344 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -151,6 +151,9 @@ install -p -m755 scripts/microshift-cleanup-data.sh %{buildroot}%{_bindir}/micro restorecon -v %{buildroot}%{_bindir}/microshift restorecon -v %{buildroot}%{_bindir}/microshift-etcd +install -d -m755 %{buildroot}{_sharedstatedir}/microshift +install -d -m755 %{buildroot}{_sharedstatedir}/microshift-backups + install -d -m755 %{buildroot}%{_sysconfdir}/crio/crio.conf.d %ifarch %{arm} aarch64 From 5483c259d6facc5230ad7f2c550974dd78ed9430 Mon Sep 17 00:00:00 2001 From: Patryk Matuszak <305846+pmtk@users.noreply.github.com> Date: Mon, 5 Jun 2023 12:01:58 +0200 Subject: [PATCH 7/7] reorganize prerun implementation --- pkg/admin/data/types.go | 2 + pkg/admin/prerun/prerun.go | 155 +++++++++++++++++++++++-------------- pkg/util/util.go | 2 +- 3 files changed, 102 insertions(+), 57 deletions(-) diff --git a/pkg/admin/data/types.go b/pkg/admin/data/types.go index d11d59d104..ecabc3df53 100644 --- a/pkg/admin/data/types.go +++ b/pkg/admin/data/types.go @@ -21,4 +21,6 @@ type Manager interface { BackupExists(BackupName) (bool, error) GetBackupPath(BackupName) string + GetBackupList() ([]BackupName, error) + RemoveBackup(BackupName) error } diff --git a/pkg/admin/prerun/prerun.go b/pkg/admin/prerun/prerun.go index cd170b2ffd..59e3da61c4 100644 --- a/pkg/admin/prerun/prerun.go +++ b/pkg/admin/prerun/prerun.go @@ -2,6 +2,7 @@ package prerun import ( "encoding/json" + "errors" "fmt" "os" "strings" @@ -12,96 +13,138 @@ import ( "k8s.io/klog/v2" ) +var ( + errHealthFileDoesNotExist = errors.New("health file does not exist") +) + +type HealthInfo struct { + Health string `json:"health"` + DeploymentID string `json:"deployment_id"` + BootID string `json:"boot_id"` +} + +func (hi *HealthInfo) BackupName() data.BackupName { + return data.BackupName(fmt.Sprintf("%s_%s", hi.DeploymentID, hi.BootID)) +} + +func (hi *HealthInfo) IsHealthy() bool { + return hi.Health == "healthy" +} + func Perform() error { - healthFile := "/var/lib/microshift-backups/health.json" - exists, err := util.PathExists(healthFile) + health, err := getHealthInfo() if err != nil { - klog.ErrorS(err, "Could not access health file", "path", healthFile) + if errors.Is(err, errHealthFileDoesNotExist) { + klog.InfoS("Health file does not exist - skipping backup") + return nil + } + klog.ErrorS(err, "Failed to load health from disk") return err } - if !exists { - klog.InfoS("Boot health information is missing - skipping backup") + klog.InfoS("Loaded health info from the disk", "health", health) + + if isCurr, err := containsCurrentBootID(health.BootID); err != nil { + return err + } else if isCurr { + klog.InfoS("Health file contains current boot - skipping backup") return nil } - currentBootID, err := getCurrentBootID() - if err != nil { - klog.ErrorS(err, "Failed to get current boot ID") - return err + if !health.IsHealthy() { + klog.InfoS("System was not healthy - skipping backup") + return nil } - klog.InfoS("Current boot", "id", currentBootID) - - info := struct { - Health string `json:"health"` - DeploymentID string `json:"deployment_id"` - BootID string `json:"boot_id"` - }{} - d, err := os.ReadFile(healthFile) + + dataManager, err := data.NewManager(config.BackupsDir) if err != nil { - klog.ErrorS(err, "Failed to read file", "path", healthFile) return err } - if err := json.Unmarshal(d, &info); err != nil { - klog.ErrorS(err, "Failed to unmarshal json", "data", string(d)) + + existingBackups, err := dataManager.GetBackupList() + if err != nil { return err } - klog.InfoS("Read health info", "info", info) + // get list of already existing backups for deployment ID persisted in health file + // after creating backup, the list will be used to remove older backups + // (so only the most recent one for specific deployment is kept) + backupsForDeployment := getExistingBackupsForTheDeployment(existingBackups, health.DeploymentID) - if info.BootID == currentBootID { - klog.InfoS("Current boot in health file - skipping backup") + newBackupName := health.BackupName() + if backupAlreadyExists(backupsForDeployment, newBackupName) { + klog.InfoS("Backup already exists", "name", newBackupName) return nil } - if info.Health != "healthy" { - return nil + if err := dataManager.Backup(newBackupName); err != nil { + return err } - name := data.BackupName(fmt.Sprintf("%s_%s", info.DeploymentID, info.BootID)) - dm, err := data.NewManager(config.BackupsDir) + removeOldBackups(dataManager, backupsForDeployment) + + return nil +} + +func containsCurrentBootID(id string) (bool, error) { + path := "/proc/sys/kernel/random/boot_id" + content, err := os.ReadFile(path) if err != nil { - return err + klog.ErrorS(err, "Failed to read file", "path", path) + return false, fmt.Errorf("reading file %s failed: %w", path, err) } + currentBootID := strings.ReplaceAll(strings.TrimSpace(string(content)), "-", "") + klog.InfoS("Comparing boot IDs", "current", currentBootID, "toCompare", id) + return id == currentBootID, nil +} - backups, err := dm.GetBackupList() - if err != nil { - return err +func getHealthInfo() (*HealthInfo, error) { + path := "/var/lib/microshift-backups/health.json" + if exists, err := util.PathExists(path); err != nil { + return nil, err + } else if !exists { + return nil, errHealthFileDoesNotExist } - existingDeploymentBackups := make([]data.BackupName, 0) - for _, b := range backups { - if name == b { - klog.InfoS("Backup already exists", "deployment", info.DeploymentID, "boot", info.BootID) - return nil - } - if strings.Contains(string(b), info.DeploymentID) { - existingDeploymentBackups = append(existingDeploymentBackups, b) - } + content, err := os.ReadFile(path) + if err != nil { + klog.ErrorS(err, "Failed to read file", "path", path) + return nil, err } - if err := dm.Backup(name); err != nil { - return err + health := &HealthInfo{} + if err := json.Unmarshal(content, &health); err != nil { + klog.ErrorS(err, "Failed to unmarshal file to json", "content", string(content)) + return nil, err } + return health, nil +} - if len(existingDeploymentBackups) > 0 { - klog.InfoS("Removing old deployment backups", - "deployment", info.DeploymentID, - "backups", existingDeploymentBackups) +func getExistingBackupsForTheDeployment(existingBackups []data.BackupName, deployID string) []data.BackupName { + existingDeploymentBackups := make([]data.BackupName, 0) - for _, b := range existingDeploymentBackups { - if err := dm.RemoveBackup(b); err != nil { - klog.ErrorS(err, "Failed to remove backup", "name", b) - } + for _, existingBackup := range existingBackups { + if strings.HasPrefix(string(existingBackup), deployID) { + existingDeploymentBackups = append(existingDeploymentBackups, existingBackup) } } - return nil + return existingDeploymentBackups } -func getCurrentBootID() (string, error) { - content, err := os.ReadFile("/proc/sys/kernel/random/boot_id") - if err != nil { - return "", err +func backupAlreadyExists(existingBackups []data.BackupName, name data.BackupName) bool { + for _, backup := range existingBackups { + if backup == name { + return true + } + } + return false +} + +func removeOldBackups(dataManager data.Manager, backups []data.BackupName) { + for _, b := range backups { + klog.InfoS("Removing older backup", "name", b) + if err := dataManager.RemoveBackup(b); err != nil { + klog.ErrorS(err, "Failed to remove backup", "name", b) + } } - return strings.ReplaceAll(strings.TrimSpace(string(content)), "-", ""), nil } diff --git a/pkg/util/util.go b/pkg/util/util.go index 61725b28d2..e7fb5d787d 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -25,7 +25,7 @@ func PathExists(path string) (bool, error) { } else if errors.Is(err, os.ErrNotExist) { return false, nil } else { - return false, err + return false, fmt.Errorf("checking if path (%s) exists failed: %w", path, err) } }