From 790889960fdf21f078e78186eebb12bfbe03d950 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 18 Mar 2026 11:06:18 +0100 Subject: [PATCH 1/3] git: add fetch-depth support Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- client/llb/git_test.go | 11 ++ client/llb/source.go | 14 ++ frontend/dockerfile/dfgitutil/git_ref.go | 9 ++ frontend/dockerfile/dfgitutil/git_ref_test.go | 17 +++ .../dockerfile/dockerfile2llb/convert_copy.go | 3 + solver/pb/attr.go | 1 + solver/pb/caps.go | 7 + source/git/identifier.go | 10 ++ source/git/identifier_test.go | 13 ++ source/git/source.go | 67 +++++++-- source/git/source_test.go | 137 ++++++++++++++++++ 11 files changed, 280 insertions(+), 9 deletions(-) diff --git a/client/llb/git_test.go b/client/llb/git_test.go index 38724981928e..8b67a2b266af 100644 --- a/client/llb/git_test.go +++ b/client/llb/git_test.go @@ -69,6 +69,17 @@ func TestGit(t *testing.T) { "git.fullurl": "https://github.com/foo/bar.git", }, }, + { + name: "fetch depth", + st: Git("github.com/foo/bar.git", "ref", GitFetchDepth(0)), + identifier: "git://github.com/foo/bar.git#ref", + attrs: map[string]string{ + "git.authheadersecret": "GIT_AUTH_HEADER", + "git.authtokensecret": "GIT_AUTH_TOKEN", + "git.fetchdepth": "0", + "git.fullurl": "https://github.com/foo/bar.git", + }, + }, } for _, tc := range tcases { diff --git a/client/llb/source.go b/client/llb/source.go index 0ffeddace4c0..02a5e6371503 100644 --- a/client/llb/source.go +++ b/client/llb/source.go @@ -470,6 +470,10 @@ func Git(url, fragment string, opts ...GitOption) State { attrs[pb.AttrGitChecksum] = checksum addCap(&gi.Constraints, pb.CapSourceGitChecksum) } + if gi.FetchDepth != nil { + attrs[pb.AttrGitFetchDepth] = strconv.Itoa(*gi.FetchDepth) + addCap(&gi.Constraints, pb.CapSourceGitFetchDepth) + } if gi.SkipSubmodules { attrs[pb.AttrGitSkipSubmodules] = "true" @@ -500,6 +504,7 @@ type GitInfo struct { KnownSSHHosts string MountSSHSock string Checksum string + FetchDepth *int Ref string SubDir string SkipSubmodules bool @@ -517,6 +522,15 @@ func GitSubDir(v string) GitOption { }) } +func GitFetchDepth(v int) GitOption { + return gitOptionFunc(func(gi *GitInfo) { + if v < 0 { + return + } + gi.FetchDepth = &v + }) +} + func GitSkipSubmodules() GitOption { return gitOptionFunc(func(gi *GitInfo) { gi.SkipSubmodules = true diff --git a/frontend/dockerfile/dfgitutil/git_ref.go b/frontend/dockerfile/dfgitutil/git_ref.go index a45c9fb7d5bc..4fd3e212bfaa 100644 --- a/frontend/dockerfile/dfgitutil/git_ref.go +++ b/frontend/dockerfile/dfgitutil/git_ref.go @@ -56,6 +56,9 @@ type GitRef struct { // Submodules is true for URL that controls whether to fetch git submodules. Submodules *bool + + // FetchDepth controls how much history to fetch. + FetchDepth *int } // ParseGitRef parses a git ref. @@ -158,6 +161,12 @@ func (gf *GitRef) loadQuery(query url.Values) error { gf.SubDir = v[0] case "checksum", "commit": gf.Checksum = v[0] + case "fetch-depth": + vv, err := strconv.Atoi(v[0]) + if err != nil || vv < 0 { + return errors.Errorf("invalid fetch-depth value: %q", v[0]) + } + gf.FetchDepth = &vv case "keep-git-dir": var vv bool if len(v) == 0 { diff --git a/frontend/dockerfile/dfgitutil/git_ref_test.go b/frontend/dockerfile/dfgitutil/git_ref_test.go index c767e5f1c7b2..94b66c583f76 100644 --- a/frontend/dockerfile/dfgitutil/git_ref_test.go +++ b/frontend/dockerfile/dfgitutil/git_ref_test.go @@ -192,6 +192,15 @@ func TestParseGitRef(t *testing.T) { Ref: "refs/heads/v1.0", }, }, + { + ref: "https://github.com/moby/buildkit.git?fetch-depth=0#v1.2.3", + expected: &GitRef{ + Remote: "https://github.com/moby/buildkit.git", + ShortName: "buildkit", + Ref: "v1.2.3", + FetchDepth: ptrInt(0), + }, + }, { ref: "https://github.com/moby/buildkit.git?ref=v1.0.0#v1.2.3", err: "ref conflicts", @@ -218,6 +227,10 @@ func TestParseGitRef(t *testing.T) { ref: "https://github.com/moby/buildkit.git?invalid=123", err: "unexpected query \"invalid\"", }, + { + ref: "https://github.com/moby/buildkit.git?fetch-depth=-1", + err: "invalid fetch-depth value", + }, } for i, tt := range cases { t.Run(fmt.Sprintf("case%d", i+1), func(t *testing.T) { @@ -236,6 +249,10 @@ func TestParseGitRef(t *testing.T) { } } +func ptrInt(v int) *int { + return &v +} + func TestFragmentFormat(t *testing.T) { cases := []struct { ref string diff --git a/frontend/dockerfile/dockerfile2llb/convert_copy.go b/frontend/dockerfile/dockerfile2llb/convert_copy.go index 37c3d4e7eb55..adf9ec404018 100644 --- a/frontend/dockerfile/dockerfile2llb/convert_copy.go +++ b/frontend/dockerfile/dockerfile2llb/convert_copy.go @@ -164,6 +164,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { if gitRef.SubDir != "" { gitOptions = append(gitOptions, llb.GitSubDir(gitRef.SubDir)) } + if gitRef.FetchDepth != nil { + gitOptions = append(gitOptions, llb.GitFetchDepth(*gitRef.FetchDepth)) + } if gitRef.Submodules != nil && !*gitRef.Submodules { gitOptions = append(gitOptions, llb.GitSkipSubmodules()) } diff --git a/solver/pb/attr.go b/solver/pb/attr.go index 0db767341bd3..14190e957f6f 100644 --- a/solver/pb/attr.go +++ b/solver/pb/attr.go @@ -7,6 +7,7 @@ const AttrAuthTokenSecret = "git.authtokensecret" const AttrKnownSSHHosts = "git.knownsshhosts" const AttrMountSSHSock = "git.mountsshsock" const AttrGitChecksum = "git.checksum" +const AttrGitFetchDepth = "git.fetchdepth" const AttrGitSkipSubmodules = "git.skipsubmodules" const AttrGitSignatureVerifyPubKey = "git.sig.pubkey" diff --git a/solver/pb/caps.go b/solver/pb/caps.go index d869b9f8152b..1fa57986bde7 100644 --- a/solver/pb/caps.go +++ b/solver/pb/caps.go @@ -32,6 +32,7 @@ const ( CapSourceGitMountSSHSock apicaps.CapID = "source.git.mountsshsock" CapSourceGitSubdir apicaps.CapID = "source.git.subdir" CapSourceGitChecksum apicaps.CapID = "source.git.checksum" + CapSourceGitFetchDepth apicaps.CapID = "source.git.fetchdepth" CapSourceGitSkipSubmodules apicaps.CapID = "source.git.skipsubmodules" CapSourceGitSignatureVerify apicaps.CapID = "source.git.signatureverify" @@ -243,6 +244,12 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapSourceGitFetchDepth, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapSourceGitSkipSubmodules, Enabled: true, diff --git a/source/git/identifier.go b/source/git/identifier.go index 02f490185973..ece293d161ac 100644 --- a/source/git/identifier.go +++ b/source/git/identifier.go @@ -2,18 +2,21 @@ package git import ( "path" + "strconv" "github.com/moby/buildkit/solver/llbsolver/provenance" provenancetypes "github.com/moby/buildkit/solver/llbsolver/provenance/types" "github.com/moby/buildkit/source" srctypes "github.com/moby/buildkit/source/types" "github.com/moby/buildkit/util/gitutil" + "github.com/pkg/errors" ) type GitIdentifier struct { Remote string Ref string Checksum string + FetchDepth *int Subdir string KeepGitDir bool AuthTokenSecret string @@ -46,6 +49,13 @@ func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) { repo.Ref = u.Opts.Ref repo.Subdir = u.Opts.Subdir } + if v := u.Query.Get("fetch-depth"); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + return nil, errors.Errorf("invalid fetch-depth value: %q", v) + } + repo.FetchDepth = &n + } if sd := path.Clean(repo.Subdir); sd == "/" || sd == "." { repo.Subdir = "" } diff --git a/source/git/identifier_test.go b/source/git/identifier_test.go index c272f581e5f0..3ecca6f1d1fe 100644 --- a/source/git/identifier_test.go +++ b/source/git/identifier_test.go @@ -72,6 +72,15 @@ func TestNewGitIdentifier(t *testing.T) { Subdir: "mydir/mysubdir/", }, }, + { + url: "https://github.com/user/repo.git?fetch-depth=0#mybranch:mydir/mysubdir/", + expected: GitIdentifier{ + Remote: "https://github.com/user/repo.git", + Ref: "mybranch", + Subdir: "mydir/mysubdir/", + FetchDepth: ptrInt(0), + }, + }, { url: "git@github.com:user/repo.git", expected: GitIdentifier{ @@ -117,3 +126,7 @@ func TestNewGitIdentifier(t *testing.T) { }) } } + +func ptrInt(v int) *int { + return &v +} diff --git a/source/git/source.go b/source/git/source.go index fe35e0f522be..c2ec40578049 100644 --- a/source/git/source.go +++ b/source/git/source.go @@ -110,6 +110,12 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo id.MountSSHSock = v case pb.AttrGitChecksum: id.Checksum = v + case pb.AttrGitFetchDepth: + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + return nil, errors.Errorf("invalid fetch-depth value: %q", v) + } + id.FetchDepth = &n case pb.AttrGitSkipSubmodules: if v == "true" { id.SkipSubmodules = true @@ -257,6 +263,9 @@ func (gs *gitSourceHandler) shaToCacheKey(sha, ref string) string { if ref != "" { key += "#" + ref } + if gs.src.FetchDepth != nil && *gs.src.FetchDepth != 1 { + key += fmt.Sprintf("(fetch-depth=%d)", *gs.src.FetchDepth) + } } if gs.src.Subdir != "" { key += ":" + gs.src.Subdir @@ -867,7 +876,7 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, if gitutil.IsCommitSHA(ref) { // skip fetch if commit already exists if _, err := git.Run(ctx, "cat-file", "-e", ref+"^{commit}"); err == nil { - doFetch = false + doFetch = gs.src.FetchDepth != nil && *gs.src.FetchDepth == 0 } } @@ -884,12 +893,32 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, os.RemoveAll(filepath.Join(gitDir, "shallow.lock")) args := []string{"fetch"} - if !gitutil.IsCommitSHA(ref) { // TODO: find a branch from ls-remote? - args = append(args, "--depth=1", "--no-tags") - } else { + if gitutil.IsCommitSHA(ref) { args = append(args, "--tags") - if _, err := os.Lstat(filepath.Join(gitDir, "shallow")); err == nil { - args = append(args, "--unshallow") + switch { + case gs.src.FetchDepth != nil && *gs.src.FetchDepth == 0: + if _, err := os.Lstat(filepath.Join(gitDir, "shallow")); err == nil { + args = append(args, "--unshallow") + } + case gs.src.FetchDepth != nil && *gs.src.FetchDepth > 0: + args = append(args, "--depth="+strconv.Itoa(*gs.src.FetchDepth)) + default: + if _, err := os.Lstat(filepath.Join(gitDir, "shallow")); err == nil { + args = append(args, "--unshallow") + } + } + } else { // TODO: find a branch from ls-remote? + if gs.src.FetchDepth != nil && *gs.src.FetchDepth == 0 { + args = append(args, "--tags") + if _, err := os.Lstat(filepath.Join(gitDir, "shallow")); err == nil { + args = append(args, "--unshallow") + } + } else { + fetchDepth := 1 + if gs.src.FetchDepth != nil { + fetchDepth = *gs.src.FetchDepth + } + args = append(args, "--depth="+strconv.Itoa(fetchDepth), "--no-tags") } } args = append(args, "origin") @@ -935,8 +964,17 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, } else { // try to fetch the commit directly args := []string{"fetch", "--tags"} - if _, err := os.Lstat(filepath.Join(gitDir, "shallow")); err == nil { - args = append(args, "--unshallow") + switch { + case gs.src.FetchDepth != nil && *gs.src.FetchDepth == 0: + if _, err := os.Lstat(filepath.Join(gitDir, "shallow")); err == nil { + args = append(args, "--unshallow") + } + case gs.src.FetchDepth != nil && *gs.src.FetchDepth > 0: + args = append(args, "--depth="+strconv.Itoa(*gs.src.FetchDepth)) + default: + if _, err := os.Lstat(filepath.Join(gitDir, "shallow")); err == nil { + args = append(args, "--unshallow") + } } args = append(args, "origin", gs.cacheCommit) if _, err := git.Run(ctx, args...); err != nil { @@ -1043,7 +1081,18 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi } else { pullref += ":" + pullref } - _, err = checkoutGit.Run(ctx, "fetch", "-u", "--depth=1", "origin", pullref) + fetchArgs := []string{"fetch", "-u"} + if gs.src.FetchDepth != nil && *gs.src.FetchDepth == 0 { + fetchArgs = append(fetchArgs, "--tags") + } else { + fetchDepth := 1 + if gs.src.FetchDepth != nil { + fetchDepth = *gs.src.FetchDepth + } + fetchArgs = append(fetchArgs, "--depth="+strconv.Itoa(fetchDepth)) + } + fetchArgs = append(fetchArgs, "origin", pullref) + _, err = checkoutGit.Run(ctx, fetchArgs...) if err != nil { return nil, err } diff --git a/source/git/source_test.go b/source/git/source_test.go index b0e2e6360282..f29e7ef0c01c 100644 --- a/source/git/source_test.go +++ b/source/git/source_test.go @@ -57,6 +57,21 @@ func TestRepeatedFetchKeepGitDirSHA256(t *testing.T) { testRepeatedFetch(t, true, "sha256") } +func TestFetchDepthKeepGitDirSHA1(t *testing.T) { + testFetchDepthKeepGitDir(t, "sha1") +} + +func TestFetchDepthKeepGitDirSHA256(t *testing.T) { + testFetchDepthKeepGitDir(t, "sha256") +} + +func TestFetchDepthPullRefKeepGitDirSHA1(t *testing.T) { + testFetchDepthPullRefKeepGitDir(t, "sha1") +} + +func TestFetchDepthPullRefKeepGitDirSHA256(t *testing.T) { + testFetchDepthPullRefKeepGitDir(t, "sha256") +} func testRepeatedFetch(t *testing.T, keepGitDir bool, format string) { if runtime.GOOS == "windows" { t.Skip("Depends on unimplemented containerd bind-mount support on Windows") @@ -175,6 +190,128 @@ func testRepeatedFetch(t *testing.T, keepGitDir bool, format string) { require.Equal(t, pin3, pin4) } +func testFetchDepthKeepGitDir(t *testing.T, format string) { + if runtime.GOOS == "windows" { + t.Skip("Depends on unimplemented containerd bind-mount support on Windows") + } + + t.Parallel() + ctx := logProgressStreams(context.Background(), t) + + gs := setupGitSource(t, t.TempDir()) + repo := setupGitRepo(t, format) + + id := &GitIdentifier{Remote: repo.mainURL, Ref: "feature", KeepGitDir: true} + g, err := gs.Resolve(ctx, id, nil, nil) + require.NoError(t, err) + + key1, _, _, done, err := g.CacheKey(ctx, nil, 0) + require.NoError(t, err) + require.True(t, done) + + ref1, err := g.Snapshot(ctx, nil) + require.NoError(t, err) + defer ref1.Release(context.TODO()) + + dir1 := mountRef(ctx, t, ref1) + requireGitDescribeFails(t, dir1, "--match", "lightweight-tag*") + _, err = os.Lstat(filepath.Join(dir1, ".git", "shallow")) + require.NoError(t, err) + + id = &GitIdentifier{Remote: repo.mainURL, Ref: "feature", KeepGitDir: true, FetchDepth: ptrInt(0)} + g, err = gs.Resolve(ctx, id, nil, nil) + require.NoError(t, err) + + key2, _, _, done, err := g.CacheKey(ctx, nil, 0) + require.NoError(t, err) + require.True(t, done) + require.NotEqual(t, key1, key2) + + ref2, err := g.Snapshot(ctx, nil) + require.NoError(t, err) + defer ref2.Release(context.TODO()) + require.NotEqual(t, ref1.ID(), ref2.ID()) + + dir2 := mountRef(ctx, t, ref2) + requireGitDescribe(t, dir2, "lightweight-tag-2-g", "--match", "lightweight-tag*") + _, err = os.Lstat(filepath.Join(dir2, ".git", "shallow")) + require.ErrorIs(t, err, os.ErrNotExist) +} + +func testFetchDepthPullRefKeepGitDir(t *testing.T, format string) { + if runtime.GOOS == "windows" { + t.Skip("Depends on unimplemented containerd bind-mount support on Windows") + } + + t.Parallel() + ctx := logProgressStreams(context.Background(), t) + + gs := setupGitSource(t, t.TempDir()) + repo := setupGitRepo(t, format) + + runShell(t, repo.mainPath, "git update-ref refs/pull/1544/merge $(git rev-parse feature)") + + id := &GitIdentifier{Remote: repo.mainURL, Ref: "refs/pull/1544/merge", KeepGitDir: true} + g, err := gs.Resolve(ctx, id, nil, nil) + require.NoError(t, err) + + ref1, err := g.Snapshot(ctx, nil) + require.NoError(t, err) + defer ref1.Release(context.TODO()) + + dir1 := mountRef(ctx, t, ref1) + requireGitDescribeFails(t, dir1, "--match", "lightweight-tag*") + + id = &GitIdentifier{Remote: repo.mainURL, Ref: "refs/pull/1544/merge", KeepGitDir: true, FetchDepth: ptrInt(0)} + g, err = gs.Resolve(ctx, id, nil, nil) + require.NoError(t, err) + + ref2, err := g.Snapshot(ctx, nil) + require.NoError(t, err) + defer ref2.Release(context.TODO()) + + dir2 := mountRef(ctx, t, ref2) + requireGitDescribe(t, dir2, "lightweight-tag-2-g", "--match", "lightweight-tag*") +} + +func mountRef(ctx context.Context, t *testing.T, ref cache.ImmutableRef) string { + t.Helper() + + mount, err := ref.Mount(ctx, true, nil) + require.NoError(t, err) + + lm := snapshot.LocalMounter(mount) + dir, err := lm.Mount() + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, lm.Unmount()) + }) + return dir +} + +func requireGitDescribe(t *testing.T, dir, prefix string, extraArgs ...string) { + t.Helper() + + args := append([]string{"describe", "--tags", "--long"}, extraArgs...) + cmd := exec.CommandContext(context.TODO(), "git", args...) + cmd.Dir = dir + + out, err := cmd.Output() + require.NoError(t, err) + require.Contains(t, strings.TrimSpace(string(out)), prefix) +} + +func requireGitDescribeFails(t *testing.T, dir string, extraArgs ...string) { + t.Helper() + + args := append([]string{"describe", "--tags", "--long"}, extraArgs...) + cmd := exec.CommandContext(context.TODO(), "git", args...) + cmd.Dir = dir + + out, err := cmd.CombinedOutput() + require.Error(t, err, string(out)) +} + func TestFetchBySHA1(t *testing.T) { testFetchBySHA(t, "sha1", false) } From f6e45c4d63d9ffb9169d123a3cdc9857df5d6ab8 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Wed, 18 Mar 2026 15:47:09 +0100 Subject: [PATCH 2/3] git: add fetch-tags support Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- client/llb/git_test.go | 11 +++ client/llb/source.go | 11 +++ frontend/dockerfile/dfgitutil/git_ref.go | 17 +++- frontend/dockerfile/dfgitutil/git_ref_test.go | 17 ++++ .../dockerfile/dockerfile2llb/convert_copy.go | 3 + solver/pb/attr.go | 1 + solver/pb/caps.go | 7 ++ source/git/identifier.go | 11 +++ source/git/identifier_test.go | 8 ++ source/git/source.go | 19 ++++- source/git/source_test.go | 78 +++++++++++++++++++ 11 files changed, 180 insertions(+), 3 deletions(-) diff --git a/client/llb/git_test.go b/client/llb/git_test.go index 8b67a2b266af..e64a0e5c8a33 100644 --- a/client/llb/git_test.go +++ b/client/llb/git_test.go @@ -80,6 +80,17 @@ func TestGit(t *testing.T) { "git.fullurl": "https://github.com/foo/bar.git", }, }, + { + name: "fetch tags", + st: Git("github.com/foo/bar.git", "ref", GitFetchTags()), + identifier: "git://github.com/foo/bar.git#ref", + attrs: map[string]string{ + "git.authheadersecret": "GIT_AUTH_HEADER", + "git.authtokensecret": "GIT_AUTH_TOKEN", + "git.fetchtags": "true", + "git.fullurl": "https://github.com/foo/bar.git", + }, + }, } for _, tc := range tcases { diff --git a/client/llb/source.go b/client/llb/source.go index 02a5e6371503..2d64a3356e3a 100644 --- a/client/llb/source.go +++ b/client/llb/source.go @@ -474,6 +474,10 @@ func Git(url, fragment string, opts ...GitOption) State { attrs[pb.AttrGitFetchDepth] = strconv.Itoa(*gi.FetchDepth) addCap(&gi.Constraints, pb.CapSourceGitFetchDepth) } + if gi.FetchTags { + attrs[pb.AttrGitFetchTags] = "true" + addCap(&gi.Constraints, pb.CapSourceGitFetchTags) + } if gi.SkipSubmodules { attrs[pb.AttrGitSkipSubmodules] = "true" @@ -505,6 +509,7 @@ type GitInfo struct { MountSSHSock string Checksum string FetchDepth *int + FetchTags bool Ref string SubDir string SkipSubmodules bool @@ -531,6 +536,12 @@ func GitFetchDepth(v int) GitOption { }) } +func GitFetchTags() GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.FetchTags = true + }) +} + func GitSkipSubmodules() GitOption { return gitOptionFunc(func(gi *GitInfo) { gi.SkipSubmodules = true diff --git a/frontend/dockerfile/dfgitutil/git_ref.go b/frontend/dockerfile/dfgitutil/git_ref.go index 4fd3e212bfaa..682cc2878637 100644 --- a/frontend/dockerfile/dfgitutil/git_ref.go +++ b/frontend/dockerfile/dfgitutil/git_ref.go @@ -59,6 +59,9 @@ type GitRef struct { // FetchDepth controls how much history to fetch. FetchDepth *int + + // FetchTags controls whether to fetch tag refs for shallow clones. + FetchTags *bool } // ParseGitRef parses a git ref. @@ -134,7 +137,7 @@ func (gf *GitRef) loadQuery(query url.Values) error { case 0, 1: if len(v) == 0 || v[0] == "" { switch k { - case "submodules", "keep-git-dir": + case "fetch-tags", "submodules", "keep-git-dir": v = nil default: return errors.Errorf("query %q has no value", k) @@ -167,6 +170,18 @@ func (gf *GitRef) loadQuery(query url.Values) error { return errors.Errorf("invalid fetch-depth value: %q", v[0]) } gf.FetchDepth = &vv + case "fetch-tags": + var vv bool + if len(v) == 0 { + vv = true + } else { + var err error + vv, err = strconv.ParseBool(v[0]) + if err != nil { + return errors.Errorf("invalid fetch-tags value: %q", v[0]) + } + } + gf.FetchTags = &vv case "keep-git-dir": var vv bool if len(v) == 0 { diff --git a/frontend/dockerfile/dfgitutil/git_ref_test.go b/frontend/dockerfile/dfgitutil/git_ref_test.go index 94b66c583f76..40ed2dcee489 100644 --- a/frontend/dockerfile/dfgitutil/git_ref_test.go +++ b/frontend/dockerfile/dfgitutil/git_ref_test.go @@ -201,6 +201,15 @@ func TestParseGitRef(t *testing.T) { FetchDepth: ptrInt(0), }, }, + { + ref: "https://github.com/moby/buildkit.git?fetch-tags#v1.2.3", + expected: &GitRef{ + Remote: "https://github.com/moby/buildkit.git", + ShortName: "buildkit", + Ref: "v1.2.3", + FetchTags: ptrBool(true), + }, + }, { ref: "https://github.com/moby/buildkit.git?ref=v1.0.0#v1.2.3", err: "ref conflicts", @@ -231,6 +240,10 @@ func TestParseGitRef(t *testing.T) { ref: "https://github.com/moby/buildkit.git?fetch-depth=-1", err: "invalid fetch-depth value", }, + { + ref: "https://github.com/moby/buildkit.git?fetch-tags=wat", + err: "invalid fetch-tags value", + }, } for i, tt := range cases { t.Run(fmt.Sprintf("case%d", i+1), func(t *testing.T) { @@ -253,6 +266,10 @@ func ptrInt(v int) *int { return &v } +func ptrBool(v bool) *bool { + return &v +} + func TestFragmentFormat(t *testing.T) { cases := []struct { ref string diff --git a/frontend/dockerfile/dockerfile2llb/convert_copy.go b/frontend/dockerfile/dockerfile2llb/convert_copy.go index adf9ec404018..ba543585de1b 100644 --- a/frontend/dockerfile/dockerfile2llb/convert_copy.go +++ b/frontend/dockerfile/dockerfile2llb/convert_copy.go @@ -167,6 +167,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { if gitRef.FetchDepth != nil { gitOptions = append(gitOptions, llb.GitFetchDepth(*gitRef.FetchDepth)) } + if gitRef.FetchTags != nil && *gitRef.FetchTags { + gitOptions = append(gitOptions, llb.GitFetchTags()) + } if gitRef.Submodules != nil && !*gitRef.Submodules { gitOptions = append(gitOptions, llb.GitSkipSubmodules()) } diff --git a/solver/pb/attr.go b/solver/pb/attr.go index 14190e957f6f..1804de8b52e8 100644 --- a/solver/pb/attr.go +++ b/solver/pb/attr.go @@ -8,6 +8,7 @@ const AttrKnownSSHHosts = "git.knownsshhosts" const AttrMountSSHSock = "git.mountsshsock" const AttrGitChecksum = "git.checksum" const AttrGitFetchDepth = "git.fetchdepth" +const AttrGitFetchTags = "git.fetchtags" const AttrGitSkipSubmodules = "git.skipsubmodules" const AttrGitSignatureVerifyPubKey = "git.sig.pubkey" diff --git a/solver/pb/caps.go b/solver/pb/caps.go index 1fa57986bde7..986110e86a3d 100644 --- a/solver/pb/caps.go +++ b/solver/pb/caps.go @@ -33,6 +33,7 @@ const ( CapSourceGitSubdir apicaps.CapID = "source.git.subdir" CapSourceGitChecksum apicaps.CapID = "source.git.checksum" CapSourceGitFetchDepth apicaps.CapID = "source.git.fetchdepth" + CapSourceGitFetchTags apicaps.CapID = "source.git.fetchtags" CapSourceGitSkipSubmodules apicaps.CapID = "source.git.skipsubmodules" CapSourceGitSignatureVerify apicaps.CapID = "source.git.signatureverify" @@ -250,6 +251,12 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapSourceGitFetchTags, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapSourceGitSkipSubmodules, Enabled: true, diff --git a/source/git/identifier.go b/source/git/identifier.go index ece293d161ac..ee5eed3f5969 100644 --- a/source/git/identifier.go +++ b/source/git/identifier.go @@ -17,6 +17,7 @@ type GitIdentifier struct { Ref string Checksum string FetchDepth *int + FetchTags bool Subdir string KeepGitDir bool AuthTokenSecret string @@ -56,6 +57,16 @@ func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) { } repo.FetchDepth = &n } + if vals, ok := u.Query["fetch-tags"]; ok { + repo.FetchTags = true + if len(vals) > 0 && vals[0] != "" { + v, err := strconv.ParseBool(vals[0]) + if err != nil { + return nil, errors.Errorf("invalid fetch-tags value: %q", vals[0]) + } + repo.FetchTags = v + } + } if sd := path.Clean(repo.Subdir); sd == "/" || sd == "." { repo.Subdir = "" } diff --git a/source/git/identifier_test.go b/source/git/identifier_test.go index 3ecca6f1d1fe..c2a016e80f85 100644 --- a/source/git/identifier_test.go +++ b/source/git/identifier_test.go @@ -81,6 +81,14 @@ func TestNewGitIdentifier(t *testing.T) { FetchDepth: ptrInt(0), }, }, + { + url: "https://github.com/user/repo.git?fetch-tags#mybranch", + expected: GitIdentifier{ + Remote: "https://github.com/user/repo.git", + Ref: "mybranch", + FetchTags: true, + }, + }, { url: "git@github.com:user/repo.git", expected: GitIdentifier{ diff --git a/source/git/source.go b/source/git/source.go index c2ec40578049..94d94204310d 100644 --- a/source/git/source.go +++ b/source/git/source.go @@ -116,6 +116,10 @@ func (gs *Source) Identifier(scheme, ref string, attrs map[string]string, platfo return nil, errors.Errorf("invalid fetch-depth value: %q", v) } id.FetchDepth = &n + case pb.AttrGitFetchTags: + if v == "true" { + id.FetchTags = true + } case pb.AttrGitSkipSubmodules: if v == "true" { id.SkipSubmodules = true @@ -266,6 +270,9 @@ func (gs *gitSourceHandler) shaToCacheKey(sha, ref string) string { if gs.src.FetchDepth != nil && *gs.src.FetchDepth != 1 { key += fmt.Sprintf("(fetch-depth=%d)", *gs.src.FetchDepth) } + if gs.src.FetchTags && (gs.src.FetchDepth == nil || *gs.src.FetchDepth != 0) { + key += "(fetch-tags=true)" + } } if gs.src.Subdir != "" { key += ":" + gs.src.Subdir @@ -876,7 +883,7 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, if gitutil.IsCommitSHA(ref) { // skip fetch if commit already exists if _, err := git.Run(ctx, "cat-file", "-e", ref+"^{commit}"); err == nil { - doFetch = gs.src.FetchDepth != nil && *gs.src.FetchDepth == 0 + doFetch = gs.src.FetchTags || (gs.src.FetchDepth != nil && *gs.src.FetchDepth == 0) } } @@ -918,7 +925,12 @@ func (gs *gitSourceHandler) tryRemoteFetch(ctx context.Context, g session.Group, if gs.src.FetchDepth != nil { fetchDepth = *gs.src.FetchDepth } - args = append(args, "--depth="+strconv.Itoa(fetchDepth), "--no-tags") + args = append(args, "--depth="+strconv.Itoa(fetchDepth)) + if gs.src.FetchTags { + args = append(args, "--tags") + } else { + args = append(args, "--no-tags") + } } } args = append(args, "origin") @@ -1090,6 +1102,9 @@ func (gs *gitSourceHandler) checkout(ctx context.Context, repo *gitRepo, g sessi fetchDepth = *gs.src.FetchDepth } fetchArgs = append(fetchArgs, "--depth="+strconv.Itoa(fetchDepth)) + if gs.src.FetchTags { + fetchArgs = append(fetchArgs, "--tags") + } } fetchArgs = append(fetchArgs, "origin", pullref) _, err = checkoutGit.Run(ctx, fetchArgs...) diff --git a/source/git/source_test.go b/source/git/source_test.go index f29e7ef0c01c..c7dcb677f980 100644 --- a/source/git/source_test.go +++ b/source/git/source_test.go @@ -65,6 +65,14 @@ func TestFetchDepthKeepGitDirSHA256(t *testing.T) { testFetchDepthKeepGitDir(t, "sha256") } +func TestFetchTagsKeepGitDirSHA1(t *testing.T) { + testFetchTagsKeepGitDir(t, "sha1") +} + +func TestFetchTagsKeepGitDirSHA256(t *testing.T) { + testFetchTagsKeepGitDir(t, "sha256") +} + func TestFetchDepthPullRefKeepGitDirSHA1(t *testing.T) { testFetchDepthPullRefKeepGitDir(t, "sha1") } @@ -72,6 +80,7 @@ func TestFetchDepthPullRefKeepGitDirSHA1(t *testing.T) { func TestFetchDepthPullRefKeepGitDirSHA256(t *testing.T) { testFetchDepthPullRefKeepGitDir(t, "sha256") } + func testRepeatedFetch(t *testing.T, keepGitDir bool, format string) { if runtime.GOOS == "windows" { t.Skip("Depends on unimplemented containerd bind-mount support on Windows") @@ -238,6 +247,54 @@ func testFetchDepthKeepGitDir(t *testing.T, format string) { require.ErrorIs(t, err, os.ErrNotExist) } +func testFetchTagsKeepGitDir(t *testing.T, format string) { + if runtime.GOOS == "windows" { + t.Skip("Depends on unimplemented containerd bind-mount support on Windows") + } + + t.Parallel() + ctx := logProgressStreams(context.Background(), t) + + gs := setupGitSource(t, t.TempDir()) + repo := setupGitRepo(t, format) + + id := &GitIdentifier{Remote: repo.mainURL, Ref: "feature", KeepGitDir: true} + g, err := gs.Resolve(ctx, id, nil, nil) + require.NoError(t, err) + + key1, _, _, done, err := g.CacheKey(ctx, nil, 0) + require.NoError(t, err) + require.True(t, done) + ref1, err := g.Snapshot(ctx, nil) + require.NoError(t, err) + defer ref1.Release(context.TODO()) + + dir1 := mountRef(ctx, t, ref1) + requireGitRevParseFails(t, dir1, "refs/tags/lightweight-tag") + _, err = os.Lstat(filepath.Join(dir1, ".git", "shallow")) + require.NoError(t, err) + + id = &GitIdentifier{Remote: repo.mainURL, Ref: "feature", KeepGitDir: true, FetchTags: true} + g, err = gs.Resolve(ctx, id, nil, nil) + require.NoError(t, err) + + key2, _, _, done, err := g.CacheKey(ctx, nil, 0) + require.NoError(t, err) + require.True(t, done) + require.NotEqual(t, key1, key2) + + ref2, err := g.Snapshot(ctx, nil) + require.NoError(t, err) + defer ref2.Release(context.TODO()) + require.NotEqual(t, ref1.ID(), ref2.ID()) + + dir2 := mountRef(ctx, t, ref2) + requireGitRevParse(t, dir2, "refs/tags/lightweight-tag") + requireGitDescribeFails(t, dir2, "--match", "lightweight-tag*") + _, err = os.Lstat(filepath.Join(dir2, ".git", "shallow")) + require.NoError(t, err) +} + func testFetchDepthPullRefKeepGitDir(t *testing.T, format string) { if runtime.GOOS == "windows" { t.Skip("Depends on unimplemented containerd bind-mount support on Windows") @@ -312,6 +369,27 @@ func requireGitDescribeFails(t *testing.T, dir string, extraArgs ...string) { require.Error(t, err, string(out)) } +func requireGitRevParse(t *testing.T, dir, ref string) { + t.Helper() + + cmd := exec.CommandContext(context.TODO(), "git", "rev-parse", ref) + cmd.Dir = dir + + out, err := cmd.Output() + require.NoError(t, err) + require.NotEmpty(t, strings.TrimSpace(string(out))) +} + +func requireGitRevParseFails(t *testing.T, dir, ref string) { + t.Helper() + + cmd := exec.CommandContext(context.TODO(), "git", "rev-parse", ref) + cmd.Dir = dir + + out, err := cmd.CombinedOutput() + require.Error(t, err, string(out)) +} + func TestFetchBySHA1(t *testing.T) { testFetchBySHA(t, "sha1", false) } From 35318a015d82c4a1a0463fa41e2bb529d8e4a13f Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Wed, 18 Mar 2026 12:14:56 +0100 Subject: [PATCH 3/3] dockerui: forward git fetch options for remote contexts Signed-off-by: CrazyMax <1951866+crazy-max@users.noreply.github.com> --- frontend/dockerui/build_test.go | 65 +++++++++++++++++++++++++++++++++ frontend/dockerui/context.go | 6 +++ 2 files changed, 71 insertions(+) diff --git a/frontend/dockerui/build_test.go b/frontend/dockerui/build_test.go index 378f209b285f..3ecb4cf04ffa 100644 --- a/frontend/dockerui/build_test.go +++ b/frontend/dockerui/build_test.go @@ -1,10 +1,14 @@ package dockerui import ( + "context" "testing" "github.com/containerd/platforms" + "github.com/moby/buildkit/client/llb" "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/solver/pb" + digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/require" ) @@ -98,3 +102,64 @@ func TestNormalizePlatform(t *testing.T) { require.Equal(t, platforms.FormatAll(platforms.Normalize(tc.p)), tc.expected.ID) } } + +func TestDetectGitContextForwardsFetchDepth(t *testing.T) { + t.Parallel() + + st, ok, err := DetectGitContext("https://github.com/crazy-max/diun.git?ref=refs/pull/1544/merge&subdir=.&fetch-depth=0", nil) + require.True(t, ok) + require.NoError(t, err) + + g := marshalGitContext(t, st) + require.Equal(t, "git://github.com/crazy-max/diun.git#refs/pull/1544/merge:.", g.Identifier) + require.Equal(t, map[string]string{ + "git.authheadersecret": "GIT_AUTH_HEADER", + "git.authtokensecret": "GIT_AUTH_TOKEN", + "git.fetchdepth": "0", + "git.fullurl": "https://github.com/crazy-max/diun.git", + }, g.Attrs) +} + +func TestDetectGitContextForwardsFetchTags(t *testing.T) { + t.Parallel() + + st, ok, err := DetectGitContext("https://github.com/crazy-max/diun.git?ref=refs/pull/1544/merge&subdir=.&fetch-tags=true", nil) + require.True(t, ok) + require.NoError(t, err) + + g := marshalGitContext(t, st) + require.Equal(t, "git://github.com/crazy-max/diun.git#refs/pull/1544/merge:.", g.Identifier) + require.Equal(t, map[string]string{ + "git.authheadersecret": "GIT_AUTH_HEADER", + "git.authtokensecret": "GIT_AUTH_TOKEN", + "git.fetchtags": "true", + "git.fullurl": "https://github.com/crazy-max/diun.git", + }, g.Attrs) +} + +func marshalGitContext(t *testing.T, st *llb.State) *pb.SourceOp { + t.Helper() + + def, err := st.Marshal(context.TODO()) + require.NoError(t, err) + + m := map[string]*pb.Op{} + arr := make([]*pb.Op, 0, len(def.Def)) + for _, dt := range def.Def { + var op pb.Op + err := op.Unmarshal(dt) + require.NoError(t, err) + dgst := digest.FromBytes(dt) + m[string(dgst)] = &op + arr = append(arr, &op) + } + + require.Equal(t, 2, len(arr)) + + last := arr[len(arr)-1] + require.Equal(t, 1, len(last.Inputs)) + require.Equal(t, 0, int(last.Inputs[0].Index)) + require.Equal(t, m[last.Inputs[0].Digest], arr[0]) + + return arr[0].Op.(*pb.Op_Source).Source +} diff --git a/frontend/dockerui/context.go b/frontend/dockerui/context.go index 88d68569b4fc..21ea0d7c4232 100644 --- a/frontend/dockerui/context.go +++ b/frontend/dockerui/context.go @@ -161,6 +161,12 @@ func DetectGitContext(ref string, keepGit *bool) (*llb.State, bool, error) { if g.SubDir != "" { gitOpts = append(gitOpts, llb.GitSubDir(g.SubDir)) } + if g.FetchDepth != nil { + gitOpts = append(gitOpts, llb.GitFetchDepth(*g.FetchDepth)) + } + if g.FetchTags != nil && *g.FetchTags { + gitOpts = append(gitOpts, llb.GitFetchTags()) + } if g.Checksum != "" { gitOpts = append(gitOpts, llb.GitChecksum(g.Checksum)) }