diff --git a/pkg/configurer/common.go b/pkg/configurer/common.go new file mode 100644 index 00000000..e519b090 --- /dev/null +++ b/pkg/configurer/common.go @@ -0,0 +1,50 @@ +package configurer + +import ( + "encoding/json" + "fmt" + + common "github.com/Mirantis/mcc/pkg/product/common/api" + "github.com/k0sproject/rig/log" + "github.com/k0sproject/rig/os" +) + +type DockerConfigurer struct{} + +// GetDockerInfo gets docker info from the host +func (c DockerConfigurer) GetDockerInfo(h os.Host, hostKind string) (common.DockerInfo, error) { + command := "docker info --format \"{{json . }}\"" + log.Infof("%s attempting to execute `docker info`", h) + info, err := h.ExecOutput(command) + + if err != nil && hostKind != "windows" { + log.Infof("%s attempting to execute `docker info` with sudo", h) + info, err = h.ExecOutput(fmt.Sprintf("sudo %s", command)) + if err != nil { + return common.DockerInfo{}, err + } + } + + var dockerInfo common.DockerInfo + err = json.Unmarshal([]byte(info), &dockerInfo) + if err != nil { + log.Infof("%s no `docker info` found", h) + return common.DockerInfo{}, err + } + + return dockerInfo, nil +} + +// GetDockerDaemonConfig parses docker daemon json string and populate DockerDaemonConfig struct +func (c DockerConfigurer) GetDockerDaemonConfig(dockerDaemon string) (common.DockerDaemonConfig, error) { + if dockerDaemon != "" { + return common.DockerDaemonConfig{}, fmt.Errorf("the docker daemon config is empty") + } + + var config common.DockerDaemonConfig + if err := json.Unmarshal([]byte(dockerDaemon), &config); err != nil { + return common.DockerDaemonConfig{}, fmt.Errorf("failed to unmarshal json content: %w", err) + } + + return config, nil +} diff --git a/pkg/configurer/enterpriselinux/el.go b/pkg/configurer/enterpriselinux/el.go index bb771b66..101c682f 100644 --- a/pkg/configurer/enterpriselinux/el.go +++ b/pkg/configurer/enterpriselinux/el.go @@ -22,17 +22,28 @@ func (c Configurer) InstallMKEBasePackages(h os.Host) error { // UninstallMCR uninstalls docker-ee engine. func (c Configurer) UninstallMCR(h os.Host, scriptPath string, engineConfig common.MCRConfig) error { - err := h.Exec("sudo docker system prune -f") - if err != nil { - return err - } - if err := c.StopService(h, "docker"); err != nil { - return err + var err error + info, getDockerError := c.GetDockerInfo(h, c.Kind()) + if getDockerError == nil { + if err = h.Exec("sudo docker system prune -f"); err != nil { + return err + } + + if err = c.StopService(h, "docker"); err != nil { + return err + } + + if err = c.StopService(h, "containerd"); err != nil { + return err + } + + err = h.Exec("sudo yum remove -y docker-ee docker-ee-cli") } - if err := c.StopService(h, "containerd"); err != nil { - return err + if engineConfig.Prune { + c.CleanupLingeringMCR(h, info) } - return h.Exec("sudo yum remove -y docker-ee docker-ee-cli") + + return err } // InstallMCR install Docker EE engine on Linux. diff --git a/pkg/configurer/linux.go b/pkg/configurer/linux.go index c4b3d495..8245228d 100644 --- a/pkg/configurer/linux.go +++ b/pkg/configurer/linux.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" + "github.com/Mirantis/mcc/pkg/constant" "github.com/Mirantis/mcc/pkg/util" "github.com/k0sproject/rig/exec" "github.com/k0sproject/rig/os" @@ -20,6 +21,7 @@ import ( // LinuxConfigurer is a generic linux host configurer. type LinuxConfigurer struct { riglinux os.Linux + DockerConfigurer } // SbinPath is for adding sbin directories to current $PATH. @@ -278,3 +280,93 @@ func (c LinuxConfigurer) HTTPStatus(h os.Host, url string) (int, error) { return status, nil } + +// CleanupLingeringMCR removes left over MCR files after Launchpad reset. +func (c LinuxConfigurer) CleanupLingeringMCR(h os.Host, dockerInfo common.DockerInfo) { + // Use default docker root dir if not specified in docker info + dockerRootDir := constant.LinuxDefaultDockerRoot + dockerExecRootDir := constant.LinuxDefaultDockerExecRoot + dockerDaemonPath := constant.LinuxDefaultDockerDaemonPath + + // set the docker root dir from docker info if it exists + if dockerInfo != (common.DockerInfo{}) { + dockerRootDir = dockerInfo.DockerRootDir + } + + // https://docs.docker.com/config/daemon/ + if !c.riglinux.FileExist(h, dockerDaemonPath) { + // Check if the default Rootless Docker daemon config file exists + log.Debugf("%s attempting to detect Rootless docker installation", h) + // Extract the value from the XDG_CONFIG_HOME environment variable + XDG_CONFIG_HOME, err := h.ExecOutput("echo $XDG_CONFIG_HOME") + if XDG_CONFIG_HOME != "" && err == nil { + log.Debugf("%s XDG_CONFIG_HOME set to %s", h, XDG_CONFIG_HOME) + dockerDaemonPath = path.Join(strings.TrimSpace(XDG_CONFIG_HOME), "docker", "daemon.json") + } else { + dockerDaemonPath = constant.LinuxDefaultRootlessDockerDaemonPath + log.Debugf("%s XDG_CONFIG_HOME not set, using default rootless daemon path %s", h, dockerDaemonPath) + } + } + + dockerDaemonString, err := c.riglinux.ReadFile(h, dockerDaemonPath) + if err != nil { + log.Debugf("%s couldn't read the Docker Daemon config file %s: %s", h, dockerDaemonPath, err) + } + dockerConfig, err := c.GetDockerDaemonConfig(dockerDaemonString) + if err != nil { + log.Debugf("%s failed to create DockerDaemon config %s: %s", h, dockerConfig, err) + } + + if dockerConfig.Root != "" { + dockerRootDir = dockerConfig.Root + } + if dockerConfig.ExecRoot != "" { + dockerExecRootDir = dockerConfig.ExecRoot + } + + // /var/lib/ Root folder + c.attemptPathDelete(h, dockerRootDir) + if idx := strings.LastIndex(dockerRootDir, "/"); idx != -1 { + dockerRootDir = dockerRootDir[:idx] + } + dockerCriPath := path.Join(dockerRootDir, "cri-dockerd") + c.attemptPathDelete(h, dockerCriPath) + + // /var/run/ Exec-root folder + execRootNetnsUnmount := path.Join(dockerExecRootDir, "netns/default") + if err := h.Exec(fmt.Sprintf("sudo umount %s", execRootNetnsUnmount)); err != nil { + log.Debugf("%s failed to umount %s: %s", h, execRootNetnsUnmount, err) + } + + // Extras to delete if they exist + if idx := strings.LastIndex(dockerExecRootDir, "/"); idx != -1 { + dockerExecRootDir = dockerExecRootDir[:idx] + } + criDockerdMkeSock := path.Join(dockerExecRootDir, "cri-dockerd-mke.sock") + c.attemptPathDelete(h, criDockerdMkeSock) + + dockerSock := path.Join(dockerExecRootDir, "docker.sock") + c.attemptPathDelete(h, dockerSock) + + // /lib/systemd/system/ folder + c.attemptPathDelete(h, "/lib/systemd/system/cri-dockerd-mke.service") + c.attemptPathDelete(h, "/lib/systemd/system/cri-dockerd-mke.socket") +} + +func (c LinuxConfigurer) attemptPathDelete(h os.Host, path string) { + fileInfo, err := c.riglinux.Stat(h, path) + if err != nil { + log.Debugf("%s error getting file information for %s: %s", h, path, err) + } else { + command := fmt.Sprintf("sudo rm %s", path) + if fileInfo.IsDir() { + command = fmt.Sprintf("sudo rm -rf %s", path) + } + if c.riglinux.FileExist(h, path) { + if err := h.Exec(command); err != nil { + log.Infof("%s failed to remove %s: %s", h, path, err) + } + log.Infof("%s removed %s successfully", h, path) + } + } +} diff --git a/pkg/configurer/sles/sles.go b/pkg/configurer/sles/sles.go index d4825610..0e923f4c 100644 --- a/pkg/configurer/sles/sles.go +++ b/pkg/configurer/sles/sles.go @@ -25,20 +25,27 @@ func (c Configurer) InstallMKEBasePackages(h os.Host) error { // UninstallMCR uninstalls docker-ee engine. func (c Configurer) UninstallMCR(h os.Host, scriptPath string, engineConfig common.MCRConfig) error { - err := h.Exec("sudo docker system prune -f") - if err != nil { - return err - } + var err error + info, getDockerError := c.GetDockerInfo(h, c.Kind()) + if getDockerError == nil { + if err = h.Exec("sudo docker system prune -f"); err != nil { + return err + } - if err := c.StopService(h, "docker"); err != nil { - return err - } + if err = c.StopService(h, "docker"); err != nil { + return err + } - if err := c.StopService(h, "containerd"); err != nil { - return err - } + if err = c.StopService(h, "containerd"); err != nil { + return err + } - return h.Exec("sudo zypper -n remove -y --clean-deps docker-ee docker-ee-cli") + err = h.Exec("sudo zypper -n remove -y --clean-deps docker-ee docker-ee-cli") + } + if engineConfig.Prune { + c.CleanupLingeringMCR(h, info) + } + return err } // LocalAddresses returns a list of local addresses, SLES12 has an old version of "hostname" without "--all-ip-addresses" and because of that, ip addr show is used here. diff --git a/pkg/configurer/ubuntu/ubuntu.go b/pkg/configurer/ubuntu/ubuntu.go index 516dd212..85c738d9 100644 --- a/pkg/configurer/ubuntu/ubuntu.go +++ b/pkg/configurer/ubuntu/ubuntu.go @@ -21,15 +21,25 @@ func (c Configurer) InstallMKEBasePackages(h os.Host) error { // UninstallMCR uninstalls docker-ee engine. func (c Configurer) UninstallMCR(h os.Host, scriptPath string, engineConfig common.MCRConfig) error { - err := h.Exec("sudo docker system prune -f") - if err != nil { - return err - } - if err := c.StopService(h, "docker"); err != nil { - return err + var err error + info, getDockerError := c.GetDockerInfo(h, c.Kind()) + if getDockerError == nil { + if err = h.Exec("sudo docker system prune -f"); err != nil { + return err + } + + if err = c.StopService(h, "docker"); err != nil { + return err + } + + if err = c.StopService(h, "containerd"); err != nil { + return err + } + + err = h.Exec("sudo apt-get remove -y docker-ee docker-ee-cli && sudo apt autoremove -y") } - if err := c.StopService(h, "containerd"); err != nil { - return err + if engineConfig.Prune { + c.CleanupLingeringMCR(h, info) } - return h.Exec("sudo apt-get remove -y docker-ee docker-ee-cli && sudo apt autoremove -y") + return err } diff --git a/pkg/configurer/windows.go b/pkg/configurer/windows.go index 3451ce5a..ce57930c 100644 --- a/pkg/configurer/windows.go +++ b/pkg/configurer/windows.go @@ -8,6 +8,7 @@ import ( "strings" "time" + "github.com/Mirantis/mcc/pkg/constant" common "github.com/Mirantis/mcc/pkg/product/common/api" "github.com/Mirantis/mcc/pkg/util" "github.com/avast/retry-go" @@ -24,6 +25,7 @@ type WindowsConfigurer struct { os.Windows PowerShellVersion *version.Version + DockerConfigurer } // MCRConfigPath returns the configuration file path. @@ -71,22 +73,31 @@ func (c WindowsConfigurer) InstallMCR(h os.Host, scriptPath string, engineConfig // This relies on using the http://get.mirantis.com/install.ps1 script with the '-Uninstall' option, and some cleanup as per // https://docs.microsoft.com/en-us/virtualization/windowscontainers/manage-docker/configure-docker-daemon#how-to-uninstall-docker func (c WindowsConfigurer) UninstallMCR(h os.Host, scriptPath string, engineConfig common.MCRConfig) error { - err := h.Exec(c.DockerCommandf("system prune --volumes --all -f")) - if err != nil { - return err - } + var err error + info, getDockerError := c.GetDockerInfo(h, c.Kind()) + if getDockerError == nil { + err = h.Exec(c.DockerCommandf("system prune --volumes --all -f")) + if err != nil { + return err + } - pwd := c.Pwd(h) - base := path.Base(scriptPath) - uninstaller := pwd + "\\" + base + ".ps1" - err = h.Upload(scriptPath, uninstaller) - if err != nil { - return err + pwd := c.Pwd(h) + base := path.Base(scriptPath) + uninstaller := pwd + "\\" + base + ".ps1" + err = h.Upload(scriptPath, uninstaller) + if err != nil { + return err + } + defer c.DeleteFile(h, uninstaller) + + uninstallCommand := fmt.Sprintf("powershell -NonInteractive -NoProfile -ExecutionPolicy Bypass -File %s -Uninstall -Verbose", ps.DoubleQuote(uninstaller)) + err = h.Exec(uninstallCommand) + } + if engineConfig.Prune { + c.CleanupLingeringMCR(h, info) } - defer c.DeleteFile(h, uninstaller) - uninstallCommand := fmt.Sprintf("powershell -NonInteractive -NoProfile -ExecutionPolicy Bypass -File %s -Uninstall -Verbose", ps.DoubleQuote(uninstaller)) - return h.Exec(uninstallCommand) + return err } // RestartMCR restarts Docker EE engine. @@ -234,3 +245,44 @@ func (c WindowsConfigurer) HTTPStatus(h os.Host, url string) (int, error) { func (c WindowsConfigurer) AuthorizeDocker(h os.Host) error { return nil } + +// CleanupLingeringMCR cleans up lingering MCR configuration files. +func (c WindowsConfigurer) CleanupLingeringMCR(h os.Host, dockerInfo common.DockerInfo) { + dockerRootDir := constant.WindowsDefaultDockerRoot + if dockerInfo.DockerRootDir != "" { + dockerRootDir = dockerInfo.DockerRootDir + } + + // Check if the Docker daemon configuration file exists + exists, err := h.ExecOutput(ps.Cmd(fmt.Sprintf("Test-Path %s", ps.SingleQuote(c.MCRConfigPath())))) + if err != nil { + log.Errorf("error checking if Docker Daemon configuration file exists at %s: %v", c.MCRConfigPath(), err) + } + if exists == "True" { + log.Infof("%s MCR configuration file exists at %s", h, c.MCRConfigPath()) + var dockerDaemon common.DockerDaemonConfig + dockerDaemonString, err := h.ExecOutput(ps.Cmd(fmt.Sprintf("Get-Content -Path %s", ps.SingleQuote(c.MCRConfigPath())))) + if err != nil { + dockerDaemon, err := c.DockerConfigurer.GetDockerDaemonConfig(dockerDaemonString) + if err != nil { + log.Errorf("%s error constructing dockerDaemon struct %+v: %s", h, dockerDaemon, err) + } + } + if dockerDaemon.Root != "" { + dockerRootDir = strings.TrimSpace(dockerDaemon.Root) + } + } + + c.attemptPathDelete(h, dockerRootDir) +} + +func (c WindowsConfigurer) attemptPathDelete(h os.Host, path string) { + // Remove a folder using PowerShell command. + removeCommand := fmt.Sprintf("powershell Remove-Item -LiteralPath %s -Force -Recurse ", ps.SingleQuote(path)) + + if err := h.Exec(removeCommand); err != nil { + log.Debugf("%s failed to remove %s: %s", h, path, err) + } else { + log.Infof("%s removed %s successfully", h, path) + } +} diff --git a/pkg/constant/constant.go b/pkg/constant/constant.go index 599c5cc4..c694d6d8 100644 --- a/pkg/constant/constant.go +++ b/pkg/constant/constant.go @@ -21,4 +21,14 @@ const ( ManagedLabelCmd = "node update --label-add com.mirantis.launchpad.managed=true" // ManagedMSRLabelCmd marks a MSR node as being managed by launchpad. ManagedMSRLabelCmd = "node update --label-add com.mirantis.launchpad.managed.dtr=true" + // LinuxDefaultDockerRoot defines the default docker root. + LinuxDefaultDockerRoot = "/var/lib/docker" + // LinuxDefaultDockerExecRoot defines the default docker exec root. + LinuxDefaultDockerExecRoot = "/var/run/docker" + // LinuxDefaultDockerDaemonPath defines the default docker daemon path. + LinuxDefaultDockerDaemonPath = "/etc/docker/daemon.json" + // LinuxDefaultRootlessDockerDaemonPath defines the default rootless docker daemon path. + LinuxDefaultRootlessDockerDaemonPath = "~/.config/docker/daemon.json" + // WindowsDefaultDockerRoot defines the default windows docker root. + WindowsDefaultDockerRoot = "C:\\ProgramData\\Docker" ) diff --git a/pkg/product/common/api/mcr_config.go b/pkg/product/common/api/mcr_config.go index 87fa5ac6..c50df260 100644 --- a/pkg/product/common/api/mcr_config.go +++ b/pkg/product/common/api/mcr_config.go @@ -4,6 +4,19 @@ import ( "github.com/Mirantis/mcc/pkg/constant" ) +type DockerInfo struct { + ServerVersion string `json:"ServerVersion"` + APIVersion string `json:"APIVersion"` + OS string `json:"OperatingSystem"` + KernelVersion string `json:"KernelVersion"` + DockerRootDir string `json:"DockerRootDir"` +} + +type DockerDaemonConfig struct { + ExecRoot string `json:"exec-root"` + Root string `json:"root-data"` +} + // MCRConfig holds the Mirantis Container Runtime installation specific options. type MCRConfig struct { Version string `yaml:"version"` @@ -11,6 +24,7 @@ type MCRConfig struct { InstallURLLinux string `yaml:"installURLLinux,omitempty"` InstallURLWindows string `yaml:"installURLWindows,omitempty"` Channel string `yaml:"channel,omitempty"` + Prune bool `yaml:"prune,omitempty"` } // UnmarshalYAML puts in sane defaults when unmarshaling from yaml. diff --git a/pkg/product/mke/phase/uninstall_mcr.go b/pkg/product/mke/phase/uninstall_mcr.go index 54381853..0b13d114 100644 --- a/pkg/product/mke/phase/uninstall_mcr.go +++ b/pkg/product/mke/phase/uninstall_mcr.go @@ -24,13 +24,9 @@ func (p *UninstallMCR) Run() error { } func (p *UninstallMCR) uninstallMCR(h *api.Host, c *api.ClusterConfig) error { - err := h.Exec(h.Configurer.DockerCommandf("info")) - if err != nil { - log.Infof("%s: container runtime not installed, skipping", h) - return nil //nolint:nilerr - } log.Infof("%s: uninstalling container runtime", h) - err = h.Configurer.UninstallMCR(h, h.Metadata.MCRInstallScript, c.Spec.MCR) + + err := h.Configurer.UninstallMCR(h, h.Metadata.MCRInstallScript, c.Spec.MCR) if err == nil { log.Infof("%s: mirantis container runtime uninstalled", h) }