diff --git a/client/llb/source.go b/client/llb/source.go index b4e4412f54a2..5d1f6f104f37 100644 --- a/client/llb/source.go +++ b/client/llb/source.go @@ -341,6 +341,11 @@ func Git(url, fragment string, opts ...GitOption) State { addCap(&gi.Constraints, pb.CapSourceGitChecksum) } + if gi.SkipSubmodules { + attrs[pb.AttrGitSkipSubmodules] = "true" + addCap(&gi.Constraints, pb.CapSourceGitSkipSubmodules) + } + addCap(&gi.Constraints, pb.CapSourceGit) source := NewSource("git://"+id, attrs, gi.Constraints) @@ -367,6 +372,7 @@ type GitInfo struct { Checksum string Ref string SubDir string + SkipSubmodules bool } func GitRef(v string) GitOption { @@ -381,6 +387,12 @@ func GitSubDir(v string) GitOption { }) } +func GitSkipSubmodules() GitOption { + return gitOptionFunc(func(gi *GitInfo) { + gi.SkipSubmodules = true + }) +} + func KeepGitDir() GitOption { return gitOptionFunc(func(gi *GitInfo) { gi.KeepGitDir = true diff --git a/frontend/dockerfile/dfgitutil/git_ref.go b/frontend/dockerfile/dfgitutil/git_ref.go index 54fb775750e1..27a93a66be58 100644 --- a/frontend/dockerfile/dfgitutil/git_ref.go +++ b/frontend/dockerfile/dfgitutil/git_ref.go @@ -3,6 +3,7 @@ package dfgitutil import ( "net/url" + "strconv" "strings" cerrdefs "github.com/containerd/errdefs" @@ -49,6 +50,12 @@ type GitRef struct { // Discouraged, although not deprecated. // Instead, consider using an encrypted TCP connection such as "git@github.com/foo/bar.git" or "https://github.com/foo/bar.git". UnencryptedTCP bool + + // KeepGitDir is true for URL that controls whether to keep the .git directory. + KeepGitDir *bool + + // Submodules is true for URL that controls whether to fetch git submodules. + Submodules *bool } // ParseGitRef parses a git ref. @@ -121,11 +128,14 @@ func (gf *GitRef) loadQuery(query url.Values) error { var tag, branch string for k, v := range query { switch len(v) { - case 0: - return errors.Errorf("query %q has no value", k) - case 1: - if v[0] == "" { - return errors.Errorf("query %q has no value", k) + case 0, 1: + if len(v) == 0 || v[0] == "" { + switch k { + case "submodules", "keep-git-dir": + v = nil + default: + return errors.Errorf("query %q has no value", k) + } } // NOP default: @@ -148,6 +158,30 @@ func (gf *GitRef) loadQuery(query url.Values) error { gf.SubDir = v[0] case "checksum", "commit": gf.Checksum = v[0] + case "keep-git-dir": + var vv bool + if len(v) == 0 { + vv = true + } else { + var err error + vv, err = strconv.ParseBool(v[0]) + if err != nil { + return errors.Errorf("invalid keep-git-dir value: %q", v[0]) + } + } + gf.KeepGitDir = &vv + case "submodules": + var vv bool + if len(v) == 0 { + vv = true + } else { + var err error + vv, err = strconv.ParseBool(v[0]) + if err != nil { + return errors.Errorf("invalid submodules value: %q", v[0]) + } + } + gf.Submodules = &vv default: return errors.Errorf("unexpected query %q", k) } diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index f439ce5efd96..665bdb399fe5 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -1520,7 +1520,15 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { llb.WithCustomName(pgName), llb.GitRef(gitRef.Ref), } - if cfg.keepGitDir { + if cfg.keepGitDir != nil && gitRef.KeepGitDir != nil { + if *cfg.keepGitDir != *gitRef.KeepGitDir { + return errors.New("inconsistent keep-git-dir configuration") + } + } + if gitRef.KeepGitDir != nil { + cfg.keepGitDir = gitRef.KeepGitDir + } + if cfg.keepGitDir != nil && *cfg.keepGitDir { gitOptions = append(gitOptions, llb.KeepGitDir()) } if cfg.checksum != "" && gitRef.Checksum != "" { @@ -1537,6 +1545,9 @@ func dispatchCopy(d *dispatchState, cfg copyConfig) error { if gitRef.SubDir != "" { gitOptions = append(gitOptions, llb.GitSubDir(gitRef.SubDir)) } + if gitRef.Submodules != nil && !*gitRef.Submodules { + gitOptions = append(gitOptions, llb.GitSkipSubmodules()) + } st := llb.Git(gitRef.Remote, "", gitOptions...) opts := append([]llb.CopyOption{&llb.CopyInfo{ @@ -1711,7 +1722,7 @@ type copyConfig struct { chown string chmod string link bool - keepGitDir bool + keepGitDir *bool checksum string parents bool location []parser.Range diff --git a/frontend/dockerfile/dockerfile_addgit_test.go b/frontend/dockerfile/dockerfile_addgit_test.go index 1c7dc1c30b1f..daffbd3fbbec 100644 --- a/frontend/dockerfile/dockerfile_addgit_test.go +++ b/frontend/dockerfile/dockerfile_addgit_test.go @@ -356,8 +356,25 @@ func testGitQueryString(t *testing.T, sb integration.Sandbox) { integration.SkipOnPlatform(t, "windows") f := getFrontend(t, sb) - gitDir, err := os.MkdirTemp("", "buildkit") + subModDir := t.TempDir() + defer os.RemoveAll(subModDir) + + err := runShell(subModDir, []string{ + "git init", + "git config --local user.email test", + "git config --local user.name test", + "echo 123 >file", + "git add file", + "git commit -m initial", + "git update-server-info", + }...) require.NoError(t, err) + + subModServer := httptest.NewServer(http.FileServer(http.Dir(filepath.Clean(subModDir)))) + defer subModServer.Close() + submodServerURL := subModServer.URL + + gitDir := t.TempDir() defer os.RemoveAll(gitDir) err = runShell(gitDir, []string{ "git init", @@ -368,13 +385,20 @@ func testGitQueryString(t *testing.T, sb integration.Sandbox) { require.NoError(t, err) err = os.WriteFile(filepath.Join(gitDir, "Dockerfile"), []byte(` +FROM scratch AS withgit +COPY .git/HEAD out + +FROM scratch as withsubmod +COPY submod/file out + FROM scratch COPY foo out `), 0600) require.NoError(t, err) err = runShell(gitDir, []string{ - "git add Dockerfile foo", + "git submodule add " + submodServerURL + "/.git submod", + "git add Dockerfile foo submod", "git commit -m initial", "git tag v0.0.1", "git branch base", @@ -426,6 +450,7 @@ COPY foo out type tcase struct { name string url string + target string expectOut string expectErr string } @@ -513,15 +538,73 @@ COPY foo out url: serverURL + "/.git?subdir=sub&ref=feature", expectOut: "subfeature\n", }, + { + name: "withgit", + url: serverURL + "/.git?keep-git-dir=true", + expectOut: commitHashLatest + "\n", + target: "withgit", + }, + { + name: "withgitandtag", + url: serverURL + "/.git?tag=v0.0.2&keep-git-dir=true", + expectOut: commitHashV2 + "\n", + target: "withgit", + }, + { + name: "withgit-default", + url: serverURL + "/.git", + expectErr: ".git/HEAD\": not found", + target: "withgit", + }, + { + name: "withgit-valueless", + url: serverURL + "/.git?keep-git-dir&submodules", + expectOut: commitHashLatest + "\n", + target: "withgit", + }, + { + name: "withgit-forbidden", + url: serverURL + "/.git?keep-git-dir=false", + expectErr: ".git/HEAD\": not found", + target: "withgit", + }, + { + name: "withsubmod", + url: serverURL + "/.git", + expectOut: "123\n", + target: "withsubmod", + }, + { + name: "withsubmodset", + url: serverURL + "/.git?submodules=true", + expectOut: "123\n", + target: "withsubmod", + }, + { + name: "withsubmodempty", + url: serverURL + "/.git?submodules", + expectOut: "123\n", + target: "withsubmod", + }, + { + name: "withoutsubmod", + url: serverURL + "/.git?submodules=false", + expectErr: "submod/file\": not found", + target: "withsubmod", + }, } for _, tc := range tcases { t.Run("context_"+tc.name, func(t *testing.T) { dest := t.TempDir() + attrs := map[string]string{ + "context": tc.url, + } + if tc.target != "" { + attrs["target"] = tc.target + } _, err = f.Solve(sb.Context(), c, client.SolveOpt{ - FrontendAttrs: map[string]string{ - "context": tc.url, - }, + FrontendAttrs: attrs, Exports: []client.ExportEntry{ { Type: client.ExporterLocal, @@ -551,15 +634,28 @@ COPY foo out for _, tc := range tcases { dockerfile2 := fmt.Sprintf(` -FROM scratch +FROM scratch AS main ADD %s /repo/ + +FROM scratch as withsubmod +COPY --from=main /repo/submod/file /repo/foo + +FROM scratch AS withgit +COPY --from=main /repo/.git/HEAD /repo/foo + +FROM main `, tc.url) inDir := integration.Tmpdir(t, fstest.CreateFile("Dockerfile", []byte(dockerfile2), 0600), ) t.Run("add_"+tc.name, func(t *testing.T) { dest := t.TempDir() + attrs := map[string]string{} + if tc.target != "" { + attrs["target"] = tc.target + } _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: attrs, Exports: []client.ExportEntry{ { Type: client.ExporterLocal, diff --git a/frontend/dockerfile/instructions/commands.go b/frontend/dockerfile/instructions/commands.go index f82f07f9d106..cef213ab0e33 100644 --- a/frontend/dockerfile/instructions/commands.go +++ b/frontend/dockerfile/instructions/commands.go @@ -244,7 +244,7 @@ type AddCommand struct { Chmod string Link bool ExcludePatterns []string - KeepGitDir bool // whether to keep .git dir, only meaningful for git sources + KeepGitDir *bool // whether to keep .git dir, only meaningful for git sources Checksum string Unpack *bool } diff --git a/frontend/dockerfile/instructions/parse.go b/frontend/dockerfile/instructions/parse.go index ccb0820fa828..1f7e93da6298 100644 --- a/frontend/dockerfile/instructions/parse.go +++ b/frontend/dockerfile/instructions/parse.go @@ -354,13 +354,19 @@ func parseAdd(req parseRequest) (*AddCommand, error) { unpack = &b } + var keepGit *bool + if _, ok := req.flags.used["keep-git-dir"]; ok { + b := flKeepGitDir.Value == "true" + keepGit = &b + } + return &AddCommand{ withNameAndCode: newWithNameAndCode(req), SourcesAndDest: *sourcesAndDest, Chown: flChown.Value, Chmod: flChmod.Value, Link: flLink.Value == "true", - KeepGitDir: flKeepGitDir.Value == "true", + KeepGitDir: keepGit, Checksum: flChecksum.Value, ExcludePatterns: stringValuesFromFlagIfPossible(flExcludes), Unpack: unpack, diff --git a/frontend/dockerui/context.go b/frontend/dockerui/context.go index 506cb21fffee..88d68569b4fc 100644 --- a/frontend/dockerui/context.go +++ b/frontend/dockerui/context.go @@ -69,9 +69,9 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) { bctx.dockerfileLocalName = v } - keepGit := false + var keepGit *bool if v, err := strconv.ParseBool(opts[keyContextKeepGitDirArg]); err == nil { - keepGit = v + keepGit = &v } if st, ok, err := DetectGitContext(opts[localNameContext], keepGit); ok { if err != nil { @@ -143,7 +143,7 @@ func (bc *Client) initContext(ctx context.Context) (*buildContext, error) { return bctx, nil } -func DetectGitContext(ref string, keepGit bool) (*llb.State, bool, error) { +func DetectGitContext(ref string, keepGit *bool) (*llb.State, bool, error) { g, isGit, err := dfgitutil.ParseGitRef(ref) if err != nil { return nil, isGit, err @@ -152,7 +152,10 @@ func DetectGitContext(ref string, keepGit bool) (*llb.State, bool, error) { llb.GitRef(g.Ref), WithInternalName("load git source " + ref), } - if keepGit { + if g.KeepGitDir != nil && *g.KeepGitDir { + gitOpts = append(gitOpts, llb.KeepGitDir()) + } + if keepGit != nil && *keepGit { gitOpts = append(gitOpts, llb.KeepGitDir()) } if g.SubDir != "" { @@ -161,6 +164,9 @@ func DetectGitContext(ref string, keepGit bool) (*llb.State, bool, error) { if g.Checksum != "" { gitOpts = append(gitOpts, llb.GitChecksum(g.Checksum)) } + if g.Submodules != nil && !*g.Submodules { + gitOpts = append(gitOpts, llb.GitSkipSubmodules()) + } st := llb.Git(g.Remote, "", gitOpts...) return &st, true, nil diff --git a/frontend/dockerui/namedcontext.go b/frontend/dockerui/namedcontext.go index 78e8b1dc4fe4..2033b374958b 100644 --- a/frontend/dockerui/namedcontext.go +++ b/frontend/dockerui/namedcontext.go @@ -138,7 +138,7 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke } return &st, &img, nil case "git": - st, ok, err := DetectGitContext(nc.input, true) + st, ok, err := DetectGitContext(nc.input, nil) if !ok { return nil, nil, errors.Errorf("invalid git context %s", nc.input) } @@ -147,7 +147,7 @@ func (nc *NamedContext) load(ctx context.Context, count int) (*llb.State, *docke } return st, nil, nil case "http", "https": - st, ok, err := DetectGitContext(nc.input, true) + st, ok, err := DetectGitContext(nc.input, nil) if !ok { httpst := llb.HTTP(nc.input, llb.WithCustomName("[context "+nc.nameWithPlatform+"] "+nc.input)) st = &httpst diff --git a/solver/pb/attr.go b/solver/pb/attr.go index 1157d987750e..f45a90622b13 100644 --- a/solver/pb/attr.go +++ b/solver/pb/attr.go @@ -7,6 +7,7 @@ const AttrAuthTokenSecret = "git.authtokensecret" const AttrKnownSSHHosts = "git.knownsshhosts" const AttrMountSSHSock = "git.mountsshsock" const AttrGitChecksum = "git.checksum" +const AttrGitSkipSubmodules = "git.skipsubmodules" const AttrLocalSessionID = "local.session" const AttrLocalUniqueID = "local.unique" diff --git a/solver/pb/caps.go b/solver/pb/caps.go index 75c298ae0d5c..2790c7e0f8ab 100644 --- a/solver/pb/caps.go +++ b/solver/pb/caps.go @@ -23,14 +23,15 @@ const ( CapSourceLocalDiffer apicaps.CapID = "source.local.differ" CapSourceMetadataTransfer apicaps.CapID = "source.local.metadatatransfer" - CapSourceGit apicaps.CapID = "source.git" - CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir" - CapSourceGitFullURL apicaps.CapID = "source.git.fullurl" - CapSourceGitHTTPAuth apicaps.CapID = "source.git.httpauth" - CapSourceGitKnownSSHHosts apicaps.CapID = "source.git.knownsshhosts" - CapSourceGitMountSSHSock apicaps.CapID = "source.git.mountsshsock" - CapSourceGitSubdir apicaps.CapID = "source.git.subdir" - CapSourceGitChecksum apicaps.CapID = "source.git.checksum" + CapSourceGit apicaps.CapID = "source.git" + CapSourceGitKeepDir apicaps.CapID = "source.git.keepgitdir" + CapSourceGitFullURL apicaps.CapID = "source.git.fullurl" + CapSourceGitHTTPAuth apicaps.CapID = "source.git.httpauth" + CapSourceGitKnownSSHHosts apicaps.CapID = "source.git.knownsshhosts" + CapSourceGitMountSSHSock apicaps.CapID = "source.git.mountsshsock" + CapSourceGitSubdir apicaps.CapID = "source.git.subdir" + CapSourceGitChecksum apicaps.CapID = "source.git.checksum" + CapSourceGitSkipSubmodules apicaps.CapID = "source.git.skipsubmodules" CapSourceHTTP apicaps.CapID = "source.http" CapSourceHTTPAuth apicaps.CapID = "source.http.auth" @@ -229,6 +230,12 @@ func init() { Status: apicaps.CapStatusExperimental, }) + Caps.Init(apicaps.Cap{ + ID: CapSourceGitSkipSubmodules, + Enabled: true, + Status: apicaps.CapStatusExperimental, + }) + Caps.Init(apicaps.Cap{ ID: CapSourceHTTP, Enabled: true, diff --git a/source/git/identifier.go b/source/git/identifier.go index 5cfe39e6bbb8..7b85fc2c438d 100644 --- a/source/git/identifier.go +++ b/source/git/identifier.go @@ -20,6 +20,7 @@ type GitIdentifier struct { AuthHeaderSecret string MountSSHSock string KnownSSHHosts string + SkipSubmodules bool } func NewGitIdentifier(remoteURL string) (*GitIdentifier, error) { diff --git a/source/git/source.go b/source/git/source.go index 7afb4980c92f..697cce2382b7 100644 --- a/source/git/source.go +++ b/source/git/source.go @@ -94,6 +94,10 @@ func (gs *gitSource) Identifier(scheme, ref string, attrs map[string]string, pla id.MountSSHSock = v case pb.AttrGitChecksum: id.Checksum = v + case pb.AttrGitSkipSubmodules: + if v == "true" { + id.SkipSubmodules = true + } } } @@ -209,6 +213,9 @@ func (gs *gitSourceHandler) shaToCacheKey(sha, ref string) string { if gs.src.Subdir != "" { key += ":" + gs.src.Subdir } + if gs.src.SkipSubmodules { + key += "(skip-submodules)" + } return key } @@ -639,9 +646,11 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out } git = git.New(gitutil.WithWorkTree(cd), gitutil.WithGitDir(gitDir)) - _, err = git.Run(ctx, "submodule", "update", "--init", "--recursive", "--depth=1") - if err != nil { - return nil, errors.Wrapf(err, "failed to update submodules for %s", urlutil.RedactCredentials(gs.src.Remote)) + if !gs.src.SkipSubmodules { + _, err = git.Run(ctx, "submodule", "update", "--init", "--recursive", "--depth=1") + if err != nil { + return nil, errors.Wrapf(err, "failed to update submodules for %s", urlutil.RedactCredentials(gs.src.Remote)) + } } if subdir != "." {