diff --git a/daemon/graphdriver/windows/windows.go b/daemon/graphdriver/windows/windows.go
index 13bfac2d2ecbf..52b0c6d84509a 100644
--- a/daemon/graphdriver/windows/windows.go
+++ b/daemon/graphdriver/windows/windows.go
@@ -6,7 +6,6 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -23,6 +22,7 @@ import (
 	"github.com/Microsoft/go-winio"
 	"github.com/Microsoft/go-winio/archive/tar"
 	"github.com/Microsoft/go-winio/backuptar"
+	"github.com/Microsoft/go-winio/vhd"
 	"github.com/Microsoft/hcsshim"
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
@@ -33,6 +33,7 @@ import (
 	"github.com/docker/docker/pkg/reexec"
 	"github.com/docker/docker/pkg/system"
 	units "github.com/docker/go-units"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows"
 )
@@ -331,7 +332,18 @@ func (d *Driver) Remove(id string) error {
 	tmpID := fmt.Sprintf("%s-removing", rID)
 	tmpLayerPath := filepath.Join(d.info.HomeDir, tmpID)
 	if err := os.Rename(layerPath, tmpLayerPath); err != nil && !os.IsNotExist(err) {
-		return err
+		if !os.IsPermission(err) {
+			return err
+		}
+		// If permission denied, it's possible that the scratch is still mounted, an
+		// artifact after a hard daemon crash for example. Worth a shot to try detaching it
+		// before retrying the rename.
+		if detachErr := vhd.DetachVhd(filepath.Join(layerPath, "sandbox.vhdx")); detachErr != nil {
+			return errors.Wrapf(err, "failed to detach VHD: %s", detachErr)
+		}
+		if renameErr := os.Rename(layerPath, tmpLayerPath); renameErr != nil && !os.IsNotExist(renameErr) {
+			return errors.Wrapf(err, "second rename attempt following detach failed: %s", renameErr)
+		}
 	}
 	if err := hcsshim.DestroyLayer(d.info, tmpID); err != nil {
 		logrus.Errorf("Failed to DestroyLayer %s: %s", id, err)