diff --git a/builder/node.go b/builder/node.go
index ee600017578a..b1bccbf77b83 100644
--- a/builder/node.go
+++ b/builder/node.go
@@ -122,7 +122,6 @@ func (b *Builder) LoadNodes(ctx context.Context, opts ...LoadNodesOption) (_ []N
 					Name:            driver.BuilderName(n.Name),
 					EndpointAddr:    n.Endpoint,
 					DockerAPI:       dockerapi,
-					DockerContext:   b.opts.dockerCli.CurrentContext(),
 					ContextStore:    b.opts.dockerCli.ContextStore(),
 					BuildkitdFlags:  n.BuildkitdFlags,
 					Files:           n.Files,
diff --git a/driver/docker-container/driver.go b/driver/docker-container/driver.go
index 8e7248eab17e..d48f673f78d5 100644
--- a/driver/docker-container/driver.go
+++ b/driver/docker-container/driver.go
@@ -20,6 +20,7 @@ import (
 	"github.com/docker/buildx/util/imagetools"
 	"github.com/docker/buildx/util/progress"
 	"github.com/docker/cli/cli/context/docker"
+	contextstore "github.com/docker/cli/cli/context/store"
 	"github.com/docker/cli/opts"
 	"github.com/moby/buildkit/client"
 	mobyarchive "github.com/moby/go-archive"
@@ -153,19 +154,15 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
 		// is a local socket as requesting GPU on container builder creation
 		// is not enough when generating the CDI specification for GPU devices.
 		// https://github.com/docker/buildx/pull/3320
-		if os.Getenv("WSL_DISTRO_NAME") != "" {
-			if cm, err := d.ContextStore.GetMetadata(d.DockerContext); err == nil {
-				if epm, err := docker.EndpointFromContext(cm); err == nil && isSocket(epm.Host) {
-					wslLibPath := "/usr/lib/wsl"
-					if st, err := os.Stat(wslLibPath); err == nil && st.IsDir() {
-						mounts = append(mounts, mount.Mount{
-							Type:     mount.TypeBind,
-							Source:   wslLibPath,
-							Target:   wslLibPath,
-							ReadOnly: true,
-						})
-					}
-				}
+		if os.Getenv("WSL_DISTRO_NAME") != "" && hasLocalSocketEndpoint(d.EndpointAddr, d.ContextStore) {
+			wslLibPath := "/usr/lib/wsl"
+			if st, err := os.Stat(wslLibPath); err == nil && st.IsDir() {
+				mounts = append(mounts, mount.Mount{
+					Type:     mount.TypeBind,
+					Source:   wslLibPath,
+					Target:   wslLibPath,
+					ReadOnly: true,
+				})
 			}
 		}
 		hc.Mounts = mounts
@@ -574,6 +571,24 @@ func getBuildkitFlags(initConfig driver.InitConfig) []string {
 	return flags
 }
 
+func hasLocalSocketEndpoint(endpoint string, contextStore contextstore.Reader) bool {
+	if isSocket(endpoint) {
+		return true
+	}
+	if contextStore == nil {
+		return false
+	}
+	cm, err := contextStore.GetMetadata(endpoint)
+	if err != nil {
+		return false
+	}
+	epm, err := docker.EndpointFromContext(cm)
+	if err != nil {
+		return false
+	}
+	return isSocket(epm.Host)
+}
+
 func isSocket(addr string) bool {
 	switch proto, _, _ := strings.Cut(addr, "://"); proto {
 	case "unix", "npipe", "fd":
diff --git a/driver/manager.go b/driver/manager.go
index f09d22d570bc..089e6a17377b 100644
--- a/driver/manager.go
+++ b/driver/manager.go
@@ -31,7 +31,6 @@ type InitConfig struct {
 	Name            string
 	EndpointAddr    string
 	DockerAPI       dockerclient.APIClient
-	DockerContext   string
 	ContextStore    store.Reader
 	BuildkitdFlags  []string
 	Files           map[string][]byte