diff --git a/bake/bake.go b/bake/bake.go
index a8c49bf2b0f3..1e8e54f414c0 100644
--- a/bake/bake.go
+++ b/bake/bake.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"maps"
+	"net/url"
 	"os"
 	"path"
 	"path/filepath"
@@ -29,6 +30,7 @@ import (
 	hcl "github.com/hashicorp/hcl/v2"
 	"github.com/moby/buildkit/client"
 	"github.com/moby/buildkit/client/llb"
+	"github.com/moby/buildkit/frontend/dockerfile/dfgitutil"
 	"github.com/pkg/errors"
 	"github.com/zclconf/go-cty/cty"
 	"github.com/zclconf/go-cty/cty/convert"
@@ -1310,7 +1312,7 @@ func updateContext(t *build.Inputs, inp *Input) {
 		st := llb.Scratch().File(llb.Copy(*inp.State, v.Path, "/", &llb.CopyInfo{
 			CopyDirContentsOnly: true,
 		}), llb.WithCustomNamef("set context %s to %s", k, v.Path))
-		t.NamedContexts[k] = build.NamedContext{State: &st, Path: inp.URL}
+		t.NamedContexts[k] = build.NamedContext{State: &st, Path: remoteURLWithSubdir(inp.URL, v.Path)}
 	}
 
 	if t.ContextPath == "." {
@@ -1330,7 +1332,44 @@ func updateContext(t *build.Inputs, inp *Input) {
 		llb.WithCustomNamef("set context to %s", t.ContextPath),
 	)
 	t.ContextState = &st
-	t.ContextPath = inp.URL
+	t.ContextPath = remoteURLWithSubdir(inp.URL, t.ContextPath)
+}
+
+func remoteURLWithSubdir(remoteURL, subdir string) string {
+	subdir = path.Clean(subdir)
+	if subdir == "." || remoteURL == "" {
+		return remoteURL
+	}
+
+	// only relevant for git urls
+	parsed, ok, err := dfgitutil.ParseGitRef(remoteURL)
+	if err != nil || !ok {
+		return remoteURL
+	}
+	if parsed.SubDir != "" {
+		subdir = path.Clean(path.Join(parsed.SubDir, subdir))
+	}
+
+	// keep query string and transport style untouched nad only append/adjust
+	// fragment subdir
+	if !strings.Contains(remoteURL, "#") && subdir != "" {
+		if u, err := url.Parse(remoteURL); err == nil {
+			q := u.Query()
+			if q.Has("subdir") {
+				q.Set("subdir", subdir)
+				u.RawQuery = q.Encode()
+				return u.String()
+			}
+		}
+	}
+
+	// otherwise, we adjust the fragment part to add/replace subdir
+	base, frag, _ := strings.Cut(remoteURL, "#")
+	ref, _, _ := strings.Cut(frag, ":")
+	if ref == "" {
+		return base + "#:" + subdir
+	}
+	return base + "#" + ref + ":" + subdir
 }
 
 func collectLocalPaths(t build.Inputs) []string {
diff --git a/bake/bake_test.go b/bake/bake_test.go
index 1a97a9455266..b138b5c21e83 100644
--- a/bake/bake_test.go
+++ b/bake/bake_test.go
@@ -2415,6 +2415,52 @@ target "mtx" {
 	}
 }
 
+func TestRemoteURLWithSubdir(t *testing.T) {
+	tests := []struct {
+		name   string
+		remote string
+		subdir string
+		want   string
+	}{
+		{
+			name:   "git no ref",
+			remote: "https://github.com/docker/buildx.git",
+			subdir: "components/interface",
+			want:   "https://github.com/docker/buildx.git#:components/interface",
+		},
+		{
+			name:   "git with ref",
+			remote: "https://github.com/docker/buildx.git#main",
+			subdir: "components/interface",
+			want:   "https://github.com/docker/buildx.git#main:components/interface",
+		},
+		{
+			name:   "git with existing subdir",
+			remote: "https://github.com/docker/buildx.git#main:base",
+			subdir: "components/interface",
+			want:   "https://github.com/docker/buildx.git#main:base/components/interface",
+		},
+		{
+			name:   "git query ref",
+			remote: "https://github.com/docker/buildx.git?branch=main",
+			subdir: "components/interface",
+			want:   "https://github.com/docker/buildx.git?branch=main#:components/interface",
+		},
+		{
+			name:   "non git",
+			remote: "https://example.com/context.tar.gz",
+			subdir: "components/interface",
+			want:   "https://example.com/context.tar.gz",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := remoteURLWithSubdir(tt.remote, tt.subdir)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
 func stringify[V fmt.Stringer](values []V) []string {
 	s := make([]string, len(values))
 	for i, v := range values {
diff --git a/tests/bake.go b/tests/bake.go
index da3706785ab7..4daa0515f6bc 100644
--- a/tests/bake.go
+++ b/tests/bake.go
@@ -565,12 +565,12 @@ COPY super-cool.txt /
 	}{
 		{
 			name:            "no ref",
-			expectedContext: addr,
+			expectedContext: addr + "#:bar",
 		},
 		{
 			name:            "branch ref",
 			ref:             "main",
-			expectedContext: addr + "#main",
+			expectedContext: addr + "#main:bar",
 		},
 	}
 	for _, tt := range tests {