PRODENG-2480 Launchpad reset now cleans the lingering MCR files

pkg/configurer/common.go

@@ -0,0 +1,50 @@
+package configurer
+
+import (
+	"encoding/json"
+	"fmt"
+
+	common "github.com/Mirantis/mcc/pkg/product/common/api"
+	"github.com/k0sproject/rig/log"
+	"github.com/k0sproject/rig/os"
+)
+
+type DockerConfigurer struct{}
+
+// GetDockerInfo gets docker info from the host
+func (c DockerConfigurer) GetDockerInfo(h os.Host, hostKind string) (common.DockerInfo, error) {
+	command := "docker info --format \"{{json . }}\""
+	log.Infof("%s attempting to execute `docker info`", h)
+	info, err := h.ExecOutput(command)
+
+	if err != nil && hostKind != "windows" {
+		log.Infof("%s attempting to execute `docker info` with sudo", h)
+		info, err = h.ExecOutput(fmt.Sprintf("sudo %s", command))
+		if err != nil {
+			return common.DockerInfo{}, err
+		}
+	}
+
+	var dockerInfo common.DockerInfo
+	err = json.Unmarshal([]byte(info), &dockerInfo)
+	if err != nil {
+		log.Infof("%s no `docker info` found", h)
+		return common.DockerInfo{}, err
+	}
+
+	return dockerInfo, nil
+}
+
+// GetDockerDaemonConfig parses docker daemon json string and populate DockerDaemonConfig struct
+func (c DockerConfigurer) GetDockerDaemonConfig(dockerDaemon string) (common.DockerDaemonConfig, error) {
+	if dockerDaemon != "" {
+		return common.DockerDaemonConfig{}, fmt.Errorf("the docker daemon config is empty")
+	}
+
+	var config common.DockerDaemonConfig
+	if err := json.Unmarshal([]byte(dockerDaemon), &config); err != nil {
+		return common.DockerDaemonConfig{}, fmt.Errorf("failed to unmarshal json content: %w", err)
+	}
+
+	return config, nil
+}

pkg/configurer/enterpriselinux/el.go

@@ -22,17 +22,28 @@ func (c Configurer) InstallMKEBasePackages(h os.Host) error {
 
 // UninstallMCR uninstalls docker-ee engine.
 func (c Configurer) UninstallMCR(h os.Host, scriptPath string, engineConfig common.MCRConfig) error {
-	err := h.Exec("sudo docker system prune -f")
-	if err != nil {
-		return err
-	}
-	if err := c.StopService(h, "docker"); err != nil {
-		return err
+	var err error
+	info, getDockerError := c.GetDockerInfo(h, c.Kind())
+	if getDockerError == nil {
+		if err = h.Exec("sudo docker system prune -f"); err != nil {
+			return err
+		}
+
+		if err = c.StopService(h, "docker"); err != nil {
+			return err
+		}
+
+		if err = c.StopService(h, "containerd"); err != nil {
+			return err
+		}
+
+		err = h.Exec("sudo yum remove -y docker-ee docker-ee-cli")
 	}
-	if err := c.StopService(h, "containerd"); err != nil {
-		return err
+	if engineConfig.Prune {
+		c.CleanupLingeringMCR(h, info)
 	}
-	return h.Exec("sudo yum remove -y docker-ee docker-ee-cli")
+
+	return err
 }
 
 // InstallMCR install Docker EE engine on Linux.

pkg/configurer/linux.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/Mirantis/mcc/pkg/constant"
 	"github.com/Mirantis/mcc/pkg/util"
 	"github.com/k0sproject/rig/exec"
 	"github.com/k0sproject/rig/os"
@@ -20,6 +21,7 @@ import (
 // LinuxConfigurer is a generic linux host configurer.
 type LinuxConfigurer struct {
 	riglinux os.Linux
+	DockerConfigurer
 }
 
 // SbinPath is for adding sbin directories to current $PATH.
@@ -278,3 +280,93 @@ func (c LinuxConfigurer) HTTPStatus(h os.Host, url string) (int, error) {
 
 	return status, nil
 }
+
+// CleanupLingeringMCR removes left over MCR files after Launchpad reset.
+func (c LinuxConfigurer) CleanupLingeringMCR(h os.Host, dockerInfo common.DockerInfo) {
+	// Use default docker root dir if not specified in docker info
+	dockerRootDir := constant.LinuxDefaultDockerRoot
+	dockerExecRootDir := constant.LinuxDefaultDockerExecRoot
+	dockerDaemonPath := constant.LinuxDefaultDockerDaemonPath
+
+	// set the docker root dir from docker info if it exists
+	if dockerInfo != (common.DockerInfo{}) {
+		dockerRootDir = dockerInfo.DockerRootDir
+	}
+
+	// https://docs.docker.com/config/daemon/
+	if !c.riglinux.FileExist(h, dockerDaemonPath) {
+		// Check if the default Rootless Docker daemon config file exists
+		log.Debugf("%s attempting to detect Rootless docker installation", h)
+		// Extract the value from the XDG_CONFIG_HOME environment variable
+		XDG_CONFIG_HOME, err := h.ExecOutput("echo $XDG_CONFIG_HOME")
+		if XDG_CONFIG_HOME != "" && err == nil {
+			log.Debugf("%s XDG_CONFIG_HOME set to %s", h, XDG_CONFIG_HOME)
+			dockerDaemonPath = path.Join(strings.TrimSpace(XDG_CONFIG_HOME), "docker", "daemon.json")
+		} else {
+			dockerDaemonPath = constant.LinuxDefaultRootlessDockerDaemonPath
+			log.Debugf("%s XDG_CONFIG_HOME not set, using default rootless daemon path %s", h, dockerDaemonPath)
+		}
+	}
+
+	dockerDaemonString, err := c.riglinux.ReadFile(h, dockerDaemonPath)
+	if err != nil {
+		log.Debugf("%s couldn't read the Docker Daemon config file %s: %s", h, dockerDaemonPath, err)
+	}
+	dockerConfig, err := c.GetDockerDaemonConfig(dockerDaemonString)
+	if err != nil {
+		log.Debugf("%s failed to create DockerDaemon config %s: %s", h, dockerConfig, err)
+	}
+
+	if dockerConfig.Root != "" {
+		dockerRootDir = dockerConfig.Root
+	}
+	if dockerConfig.ExecRoot != "" {
+		dockerExecRootDir = dockerConfig.ExecRoot
+	}
+
+	// /var/lib/ Root folder
+	c.attemptPathDelete(h, dockerRootDir)
+	if idx := strings.LastIndex(dockerRootDir, "/"); idx != -1 {
+		dockerRootDir = dockerRootDir[:idx]
+	}
+	dockerCriPath := path.Join(dockerRootDir, "cri-dockerd")
+	c.attemptPathDelete(h, dockerCriPath)
+
+	// /var/run/ Exec-root folder
+	execRootNetnsUnmount := path.Join(dockerExecRootDir, "netns/default")
+	if err := h.Exec(fmt.Sprintf("sudo umount %s", execRootNetnsUnmount)); err != nil {
+		log.Debugf("%s failed to umount %s: %s", h, execRootNetnsUnmount, err)
+	}
+
+	// Extras to delete if they exist
+	if idx := strings.LastIndex(dockerExecRootDir, "/"); idx != -1 {
+		dockerExecRootDir = dockerExecRootDir[:idx]
+	}
+	criDockerdMkeSock := path.Join(dockerExecRootDir, "cri-dockerd-mke.sock")
+	c.attemptPathDelete(h, criDockerdMkeSock)
+
+	dockerSock := path.Join(dockerExecRootDir, "docker.sock")
+	c.attemptPathDelete(h, dockerSock)
+
+	// /lib/systemd/system/ folder
+	c.attemptPathDelete(h, "/lib/systemd/system/cri-dockerd-mke.service")
+	c.attemptPathDelete(h, "/lib/systemd/system/cri-dockerd-mke.socket")
+}
+
+func (c LinuxConfigurer) attemptPathDelete(h os.Host, path string) {
+	fileInfo, err := c.riglinux.Stat(h, path)
+	if err != nil {
+		log.Debugf("%s error getting file information for %s: %s", h, path, err)
+	} else {
+		command := fmt.Sprintf("sudo rm %s", path)
+		if fileInfo.IsDir() {
+			command = fmt.Sprintf("sudo rm -rf %s", path)
+		}
+		if c.riglinux.FileExist(h, path) {
+			if err := h.Exec(command); err != nil {
+				log.Infof("%s failed to remove %s: %s", h, path, err)
+			}
+			log.Infof("%s removed %s successfully", h, path)
+		}
+	}
+}

pkg/configurer/sles/sles.go

@@ -25,20 +25,27 @@ func (c Configurer) InstallMKEBasePackages(h os.Host) error {
 
 // UninstallMCR uninstalls docker-ee engine.
 func (c Configurer) UninstallMCR(h os.Host, scriptPath string, engineConfig common.MCRConfig) error {
-	err := h.Exec("sudo docker system prune -f")
-	if err != nil {
-		return err
-	}
+	var err error
+	info, getDockerError := c.GetDockerInfo(h, c.Kind())
+	if getDockerError == nil {
+		if err = h.Exec("sudo docker system prune -f"); err != nil {
+			return err
+		}
 
-	if err := c.StopService(h, "docker"); err != nil {
-		return err
-	}
+		if err = c.StopService(h, "docker"); err != nil {
+			return err
+		}
 
-	if err := c.StopService(h, "containerd"); err != nil {
-		return err
-	}
+		if err = c.StopService(h, "containerd"); err != nil {
+			return err
+		}
 
-	return h.Exec("sudo zypper -n remove -y --clean-deps docker-ee docker-ee-cli")
+		err = h.Exec("sudo zypper -n remove -y --clean-deps docker-ee docker-ee-cli")
+	}
+	if engineConfig.Prune {
+		c.CleanupLingeringMCR(h, info)
+	}
+	return err
 }
 
 // LocalAddresses returns a list of local addresses, SLES12 has an old version of "hostname" without "--all-ip-addresses" and because of that, ip addr show is used here.

pkg/configurer/ubuntu/ubuntu.go

@@ -21,15 +21,25 @@ func (c Configurer) InstallMKEBasePackages(h os.Host) error {
 
 // UninstallMCR uninstalls docker-ee engine.
 func (c Configurer) UninstallMCR(h os.Host, scriptPath string, engineConfig common.MCRConfig) error {
-	err := h.Exec("sudo docker system prune -f")
-	if err != nil {
-		return err
-	}
-	if err := c.StopService(h, "docker"); err != nil {
-		return err
+	var err error
+	info, getDockerError := c.GetDockerInfo(h, c.Kind())
+	if getDockerError == nil {
+		if err = h.Exec("sudo docker system prune -f"); err != nil {
+			return err
+		}
+
+		if err = c.StopService(h, "docker"); err != nil {
+			return err
+		}
+
+		if err = c.StopService(h, "containerd"); err != nil {
+			return err
+		}
+
+		err = h.Exec("sudo apt-get remove -y docker-ee docker-ee-cli && sudo apt autoremove -y")
 	}
-	if err := c.StopService(h, "containerd"); err != nil {
-		return err
+	if engineConfig.Prune {
+		c.CleanupLingeringMCR(h, info)
 	}
-	return h.Exec("sudo apt-get remove -y docker-ee docker-ee-cli && sudo apt autoremove -y")
+	return err
 }

pkg/configurer/windows.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/Mirantis/mcc/pkg/constant"
 	common "github.com/Mirantis/mcc/pkg/product/common/api"
 	"github.com/Mirantis/mcc/pkg/util"
 	"github.com/avast/retry-go"
@@ -24,6 +25,7 @@ type WindowsConfigurer struct {
 	os.Windows
 
 	PowerShellVersion *version.Version
+	DockerConfigurer
 }
 
 // MCRConfigPath returns the configuration file path.
@@ -71,22 +73,31 @@ func (c WindowsConfigurer) InstallMCR(h os.Host, scriptPath string, engineConfig
 // This relies on using the http://get.mirantis.com/install.ps1 script with the '-Uninstall' option, and some cleanup as per
 // https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-docker/configure-docker-daemon#how-to-uninstall-docker
 func (c WindowsConfigurer) UninstallMCR(h os.Host, scriptPath string, engineConfig common.MCRConfig) error {
-	err := h.Exec(c.DockerCommandf("system prune --volumes --all -f"))
-	if err != nil {
-		return err
-	}
+	var err error
+	info, getDockerError := c.GetDockerInfo(h, c.Kind())
+	if getDockerError == nil {
+		err = h.Exec(c.DockerCommandf("system prune --volumes --all -f"))
+		if err != nil {
+			return err
+		}
 
-	pwd := c.Pwd(h)
-	base := path.Base(scriptPath)
-	uninstaller := pwd + "\\" + base + ".ps1"
-	err = h.Upload(scriptPath, uninstaller)
-	if err != nil {
-		return err
+		pwd := c.Pwd(h)
+		base := path.Base(scriptPath)
+		uninstaller := pwd + "\\" + base + ".ps1"
+		err = h.Upload(scriptPath, uninstaller)
+		if err != nil {
+			return err
+		}
+		defer c.DeleteFile(h, uninstaller)
+
+		uninstallCommand := fmt.Sprintf("powershell -NonInteractive -NoProfile -ExecutionPolicy Bypass -File %s -Uninstall -Verbose", ps.DoubleQuote(uninstaller))
+		err = h.Exec(uninstallCommand)
+	}
+	if engineConfig.Prune {
+		c.CleanupLingeringMCR(h, info)
 	}
-	defer c.DeleteFile(h, uninstaller)
 
-	uninstallCommand := fmt.Sprintf("powershell -NonInteractive -NoProfile -ExecutionPolicy Bypass -File %s -Uninstall -Verbose", ps.DoubleQuote(uninstaller))
-	return h.Exec(uninstallCommand)
+	return err
 }
 
 // RestartMCR restarts Docker EE engine.
@@ -234,3 +245,44 @@ func (c WindowsConfigurer) HTTPStatus(h os.Host, url string) (int, error) {
 func (c WindowsConfigurer) AuthorizeDocker(h os.Host) error {
 	return nil
 }
+
+// CleanupLingeringMCR cleans up lingering MCR configuration files.
+func (c WindowsConfigurer) CleanupLingeringMCR(h os.Host, dockerInfo common.DockerInfo) {
+	dockerRootDir := constant.WindowsDefaultDockerRoot
+	if dockerInfo.DockerRootDir != "" {
+		dockerRootDir = dockerInfo.DockerRootDir
+	}
+
+	// Check if the Docker daemon configuration file exists
+	exists, err := h.ExecOutput(ps.Cmd(fmt.Sprintf("Test-Path %s", ps.SingleQuote(c.MCRConfigPath()))))
+	if err != nil {
+		log.Errorf("error checking if Docker Daemon configuration file exists at %s: %v", c.MCRConfigPath(), err)
+	}
+	if exists == "True" {
+		log.Infof("%s MCR configuration file exists at %s", h, c.MCRConfigPath())
+		var dockerDaemon common.DockerDaemonConfig
+		dockerDaemonString, err := h.ExecOutput(ps.Cmd(fmt.Sprintf("Get-Content -Path %s", ps.SingleQuote(c.MCRConfigPath()))))
+		if err != nil {
+			dockerDaemon, err := c.DockerConfigurer.GetDockerDaemonConfig(dockerDaemonString)
+			if err != nil {
+				log.Errorf("%s error constructing dockerDaemon struct %+v: %s", h, dockerDaemon, err)
+			}
+		}
+		if dockerDaemon.Root != "" {
+			dockerRootDir = strings.TrimSpace(dockerDaemon.Root)
+		}
+	}
+
+	c.attemptPathDelete(h, dockerRootDir)
+}
+
+func (c WindowsConfigurer) attemptPathDelete(h os.Host, path string) {
+	// Remove a folder using PowerShell command.
+	removeCommand := fmt.Sprintf("powershell Remove-Item -LiteralPath %s -Force -Recurse ", ps.SingleQuote(path))
+
+	if err := h.Exec(removeCommand); err != nil {
+		log.Debugf("%s failed to remove %s: %s", h, path, err)
+	} else {
+		log.Infof("%s removed %s successfully", h, path)
+	}
+}

pkg/constant/constant.go

@@ -21,4 +21,14 @@ const (
 	ManagedLabelCmd = "node update --label-add com.mirantis.launchpad.managed=true"
 	// ManagedMSRLabelCmd marks a MSR node as being managed by launchpad.
 	ManagedMSRLabelCmd = "node update --label-add com.mirantis.launchpad.managed.dtr=true"
+	// LinuxDefaultDockerRoot defines the default docker root.
+	LinuxDefaultDockerRoot = "/var/lib/docker"
+	// LinuxDefaultDockerExecRoot defines the default docker exec root.
+	LinuxDefaultDockerExecRoot = "/var/run/docker"
+	// LinuxDefaultDockerDaemonPath defines the default docker daemon path.
+	LinuxDefaultDockerDaemonPath = "/etc/docker/daemon.json"
+	// LinuxDefaultRootlessDockerDaemonPath defines the default rootless docker daemon path.
+	LinuxDefaultRootlessDockerDaemonPath = "~/.config/docker/daemon.json"
+	// WindowsDefaultDockerRoot defines the default windows docker root.
+	WindowsDefaultDockerRoot = "C:\\ProgramData\\Docker"
 )