From 4eadeaf0f2eef00280d00e27e418cd18ba4f1925 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Sun, 12 Dec 2021 20:39:36 -0800 Subject: [PATCH 1/3] dockerfile: add support for named contexts Stages and implicit stages from image names can be redefined with build options. This enables using more that one source directory and reusing results from other builds. This can also be used to use a local image from other build without including a registry. Contexts need to be defined as `context:name=` frontend options. The value can be image, git repository, URL, local directory or a frontend input. Signed-off-by: Tonis Tiigi --- frontend/dockerfile/builder/build.go | 75 ++++++ frontend/dockerfile/builder/caps.go | 1 + .../cmd/dockerfile-frontend/Dockerfile | 2 +- frontend/dockerfile/dockerfile2llb/convert.go | 50 +++- frontend/dockerfile/dockerfile_test.go | 242 ++++++++++++++++++ 5 files changed, 366 insertions(+), 4 deletions(-) diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index cf4cc471787f..7f7e8b42cb6d 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -14,6 +14,7 @@ import ( "strings" "github.com/containerd/containerd/platforms" + "github.com/docker/distribution/reference" "github.com/docker/go-units" controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/client/llb" @@ -461,6 +462,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { } c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url)) }, + ContextByName: contextByName(c, tp), }) if err != nil { @@ -802,6 +804,79 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c return opts } +func contextByName(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) { + return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, error) { + named, err := reference.ParseNormalizedNamed(name) + if err != nil { + return nil, nil, errors.Wrapf(err, "invalid context name %s", name) + } + name = strings.TrimSuffix(reference.FamiliarString(named), ":latest") + + opts := c.BuildOpts().Opts + v, ok := opts["context:"+name] + if !ok { + return nil, nil, nil + } + + vv := strings.SplitN(v, ":", 2) + if len(vv) != 2 { + return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name) + } + switch vv[0] { + case "docker-image": + st := llb.Image(strings.TrimPrefix(vv[1], "//"), llb.WithCustomName("[context "+name+"] "+vv[1]), llb.WithMetaResolver(c)) + return &st, nil, nil + case "git": + st, ok := detectGitContext(v, "") + if !ok { + return nil, nil, errors.Errorf("invalid git context %s", v) + } + return st, nil, nil + case "http", "https": + st, ok := detectGitContext(v, "") + if !ok { + httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v)) + st = &httpst + } + return st, nil, nil + case "local": + st := llb.Local(vv[1], llb.WithCustomName("[context "+name+"] load from client"), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint("context:"+name)) + return &st, nil, nil + case "input": + inputs, err := c.Inputs(ctx) + if err != nil { + return nil, nil, err + } + st, ok := inputs[vv[1]] + if !ok { + return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name) + } + md, ok := opts["input-metadata:"+vv[1]] + if ok { + m := make(map[string][]byte) + if err := json.Unmarshal([]byte(md), &m); err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md) + } + dt, ok := m["containerimage.config"] + if ok { + st, err = st.WithImageConfig([]byte(dt)) + if err != nil { + return nil, nil, err + } + var img dockerfile2llb.Image + if err := json.Unmarshal(dt, &img); err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name) + } + return &st, &img, nil + } + } + return &st, nil, nil + default: + return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name) + } + } +} + func wrapSource(err error, sm *llb.SourceMap, ranges []parser.Range) error { if sm == nil { return err diff --git a/frontend/dockerfile/builder/caps.go b/frontend/dockerfile/builder/caps.go index 279701154eac..3c78cd56c4da 100644 --- a/frontend/dockerfile/builder/caps.go +++ b/frontend/dockerfile/builder/caps.go @@ -12,6 +12,7 @@ import ( var enabledCaps = map[string]struct{}{ "moby.buildkit.frontend.inputs": {}, "moby.buildkit.frontend.subrequests": {}, + "moby.buildkit.frontend.contexts": {}, } func validateCaps(req string) (forward bool, err error) { diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile index b8f4e0e857df..f97d44315d74 100644 --- a/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile +++ b/frontend/dockerfile/cmd/dockerfile-frontend/Dockerfile @@ -29,7 +29,7 @@ RUN --mount=target=. --mount=type=cache,target=/root/.cache \ FROM scratch AS release LABEL moby.buildkit.frontend.network.none="true" -LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests" +LABEL moby.buildkit.frontend.caps="moby.buildkit.frontend.inputs,moby.buildkit.frontend.subrequests,moby.buildkit.frontend.contexts" COPY --from=build /dockerfile-frontend /bin/dockerfile-frontend ENTRYPOINT ["/bin/dockerfile-frontend"] diff --git a/frontend/dockerfile/dockerfile2llb/convert.go b/frontend/dockerfile/dockerfile2llb/convert.go index 0a521621ca64..218b382251aa 100644 --- a/frontend/dockerfile/dockerfile2llb/convert.go +++ b/frontend/dockerfile/dockerfile2llb/convert.go @@ -69,9 +69,20 @@ type ConvertOpt struct { SourceMap *llb.SourceMap Hostname string Warn func(short, url string, detail [][]byte, location *parser.Range) + ContextByName func(context.Context, string) (*llb.State, *Image, error) } func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, *Image, error) { + contextByName := opt.ContextByName + opt.ContextByName = func(ctx context.Context, name string) (*llb.State, *Image, error) { + if !strings.EqualFold(name, "scratch") && !strings.EqualFold(name, "context") { + if contextByName != nil { + return contextByName(ctx, name) + } + } + return nil, nil, nil + } + if len(dt) == 0 { return nil, nil, errors.Errorf("the Dockerfile cannot be empty") } @@ -133,13 +144,30 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, st.BaseName = name ds := &dispatchState{ - stage: st, deps: make(map[*dispatchState]struct{}), ctxPaths: make(map[string]struct{}), stageName: st.Name, prefixPlatform: opt.PrefixPlatform, } + if st.Name != "" { + s, img, err := opt.ContextByName(ctx, st.Name) + if err != nil { + return nil, nil, err + } + if s != nil { + ds.noinit = true + ds.state = *s + if img != nil { + ds.image = *img + } + allDispatchStates.addState(ds) + continue + } + } + + ds.stage = st + if st.Name == "" { ds.stageName = fmt.Sprintf("stage-%d", i) } @@ -237,7 +265,7 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, for i, d := range allDispatchStates.states { reachable := isReachable(target, d) // resolve image config for every stage - if d.base == nil { + if d.base == nil && !d.noinit { if d.stage.BaseName == emptyImageName { d.state = llb.Scratch() d.image = emptyImage(platformOpt.targetPlatform) @@ -260,8 +288,23 @@ func Dockerfile2LLB(ctx context.Context, dt []byte, opt ConvertOpt) (*llb.State, platform = &platformOpt.targetPlatform } d.stage.BaseName = reference.TagNameOnly(ref).String() + var isScratch bool - if metaResolver != nil && reachable { + st, img, err := opt.ContextByName(ctx, d.stage.BaseName) + if err != nil { + return err + } + if st != nil { + if img != nil { + d.image = *img + } else { + d.image = emptyImage(platformOpt.targetPlatform) + } + d.state = *st + d.platform = platform + return nil + } + if reachable { prefix := "[" if opt.PrefixPlatform && platform != nil { prefix += platforms.Format(*platform) + " " @@ -615,6 +658,7 @@ type dispatchState struct { platform *ocispecs.Platform stage instructions.Stage base *dispatchState + noinit bool deps map[*dispatchState]struct{} buildArgs []instructions.KeyValuePairOptional commands []command diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index 66856008f1c2..fa67fa5e7df6 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -39,6 +39,7 @@ import ( "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/upload/uploadprovider" "github.com/moby/buildkit/solver/errdefs" + "github.com/moby/buildkit/solver/pb" "github.com/moby/buildkit/util/contentutil" "github.com/moby/buildkit/util/testutil" "github.com/moby/buildkit/util/testutil/httpserver" @@ -118,6 +119,9 @@ var allTests = integration.TestFuncs( testShmSize, testUlimit, testCgroupParent, + testNamedImageContext, + testNamedLocalContext, + testNamedInputContext, ) var fileOpTests = integration.TestFuncs( @@ -159,6 +163,7 @@ var securityOpts []integration.TestOpt type frontend interface { Solve(context.Context, *client.Client, client.SolveOpt, chan *client.SolveStatus) (*client.SolveResponse, error) + SolveGateway(context.Context, gateway.Client, gateway.SolveRequest) (*gateway.Result, error) DFCmdArgs(string, string) (string, string) RequiresBuildctl(t *testing.T) } @@ -5406,6 +5411,222 @@ COPY --from=base /out / require.Contains(t, strings.TrimSpace(string(dt)), `/foocgroup/buildkit/`) } +func testNamedImageContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM busybox AS base +RUN cat /etc/alpine-release > /out +FROM scratch +COPY --from=base /out / +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + f := getFrontend(t, sb) + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context:busybox": "docker-image://alpine", + }, + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.True(t, len(dt) > 0) +} + +func testNamedLocalContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM busybox AS base +RUN cat /etc/alpine-release > /out +FROM scratch +COPY --from=base /out / +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + outf := []byte(`dummy-result`) + + dir2, err := tmpdir( + fstest.CreateFile("out", outf, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir2) + + f := getFrontend(t, sb) + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = f.Solve(sb.Context(), c, client.SolveOpt{ + FrontendAttrs: map[string]string{ + "context:base": "local:basedir", + }, + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + "basedir": dir2, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.True(t, len(dt) > 0) +} + +func testNamedInputContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM alpine +ENV FOO=bar +RUN echo first > /out +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + dockerfile2 := []byte(` +FROM base AS build +RUN echo "foo is $FOO" > /foo +FROM scratch +COPY --from=build /foo /out / +`) + + dir2, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile2, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + f := getFrontend(t, sb) + + b := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + res, err := f.SolveGateway(ctx, c, gateway.SolveRequest{}) + if err != nil { + return nil, err + } + ref, err := res.SingleRef() + if err != nil { + return nil, err + } + st, err := ref.ToState() + if err != nil { + return nil, err + } + + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + + dt, ok := res.Metadata["containerimage.config"] + if !ok { + return nil, errors.Errorf("no containerimage.config in metadata") + } + + dt, err = json.Marshal(map[string][]byte{ + "containerimage.config": dt, + }) + if err != nil { + return nil, err + } + + res, err = f.SolveGateway(ctx, c, gateway.SolveRequest{ + FrontendOpt: map[string]string{ + "dockerfilekey": builder.DefaultLocalNameDockerfile + "2", + "context:base": "input:base", + "input-metadata:base": string(dt), + }, + FrontendInputs: map[string]*pb.Definition{ + "base": def.ToPB(), + }, + }) + if err != nil { + return nil, err + } + return res, nil + } + + product := "buildkit_test" + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = c.Build(ctx, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + builder.DefaultLocalNameDockerfile + "2": dir2, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, product, b, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "out")) + require.NoError(t, err) + require.Equal(t, "first\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "foo")) + require.NoError(t, err) + require.Equal(t, "foo is bar\n", string(dt)) +} + func tmpdir(appliers ...fstest.Applier) (string, error) { tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile") if err != nil { @@ -5542,6 +5763,11 @@ func (f *builtinFrontend) Solve(ctx context.Context, c *client.Client, opt clien return c.Solve(ctx, nil, opt, statusChan) } +func (f *builtinFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) { + req.Frontend = "dockerfile.v0" + return c.Solve(ctx, req) +} + func (f *builtinFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) { return dfCmdArgs(ctx, dockerfile, "--frontend dockerfile.v0") } @@ -5556,6 +5782,13 @@ func (f *clientFrontend) Solve(ctx context.Context, c *client.Client, opt client return c.Build(ctx, opt, "", builder.Build, statusChan) } +func (f *clientFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) { + if req.Frontend == "" && req.Definition == nil { + req.Frontend = "dockerfile.v0" + } + return c.Solve(ctx, req) +} + func (f *clientFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) { return "", "" } @@ -5578,6 +5811,15 @@ func (f *gatewayFrontend) Solve(ctx context.Context, c *client.Client, opt clien return c.Solve(ctx, nil, opt, statusChan) } +func (f *gatewayFrontend) SolveGateway(ctx context.Context, c gateway.Client, req gateway.SolveRequest) (*gateway.Result, error) { + req.Frontend = "gateway.v0" + if req.FrontendOpt == nil { + req.FrontendOpt = make(map[string]string) + } + req.FrontendOpt["source"] = f.gw + return c.Solve(ctx, req) +} + func (f *gatewayFrontend) DFCmdArgs(ctx, dockerfile string) (string, string) { return dfCmdArgs(ctx, dockerfile, "--frontend gateway.v0 --opt=source="+f.gw) } From 20285bb53eca9c7f0b91be0375f680ae5fbfdf7f Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 13 Dec 2021 23:26:10 -0800 Subject: [PATCH 2/3] dockerfile: test named contexts with multi-platform Signed-off-by: Tonis Tiigi --- frontend/dockerfile/builder/build.go | 124 +++++++++++--------- frontend/dockerfile/dockerfile_test.go | 150 +++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 55 deletions(-) diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index 7f7e8b42cb6d..18d175ce438a 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -462,7 +462,7 @@ func Build(ctx context.Context, c client.Client) (*client.Result, error) { } c.Warn(ctx, defVtx, msg, warnOpts(sourceMap, location, detail, url)) }, - ContextByName: contextByName(c, tp), + ContextByName: contextByNameFunc(c, tp), }) if err != nil { @@ -804,7 +804,7 @@ func warnOpts(sm *llb.SourceMap, r *parser.Range, detail [][]byte, url string) c return opts } -func contextByName(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) { +func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Context, string) (*llb.State, *dockerfile2llb.Image, error) { return func(ctx context.Context, name string) (*llb.State, *dockerfile2llb.Image, error) { named, err := reference.ParseNormalizedNamed(name) if err != nil { @@ -812,68 +812,82 @@ func contextByName(c client.Client, p *ocispecs.Platform) func(context.Context, } name = strings.TrimSuffix(reference.FamiliarString(named), ":latest") - opts := c.BuildOpts().Opts - v, ok := opts["context:"+name] - if !ok { - return nil, nil, nil - } - - vv := strings.SplitN(v, ":", 2) - if len(vv) != 2 { - return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name) - } - switch vv[0] { - case "docker-image": - st := llb.Image(strings.TrimPrefix(vv[1], "//"), llb.WithCustomName("[context "+name+"] "+vv[1]), llb.WithMetaResolver(c)) - return &st, nil, nil - case "git": - st, ok := detectGitContext(v, "") - if !ok { - return nil, nil, errors.Errorf("invalid git context %s", v) - } - return st, nil, nil - case "http", "https": - st, ok := detectGitContext(v, "") - if !ok { - httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v)) - st = &httpst - } - return st, nil, nil - case "local": - st := llb.Local(vv[1], llb.WithCustomName("[context "+name+"] load from client"), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint("context:"+name)) - return &st, nil, nil - case "input": - inputs, err := c.Inputs(ctx) + if p != nil { + name := name + "::" + platforms.Format(platforms.Normalize(*p)) + st, img, err := contextByName(ctx, c, name) if err != nil { return nil, nil, err } - st, ok := inputs[vv[1]] - if !ok { - return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name) + if st != nil { + return st, img, nil + } + } + return contextByName(ctx, c, name) + } +} + +func contextByName(ctx context.Context, c client.Client, name string) (*llb.State, *dockerfile2llb.Image, error) { + opts := c.BuildOpts().Opts + v, ok := opts["context:"+name] + if !ok { + return nil, nil, nil + } + + vv := strings.SplitN(v, ":", 2) + if len(vv) != 2 { + return nil, nil, errors.Errorf("invalid context specifier %s for %s", v, name) + } + switch vv[0] { + case "docker-image": + st := llb.Image(strings.TrimPrefix(vv[1], "//"), llb.WithCustomName("[context "+name+"] "+vv[1]), llb.WithMetaResolver(c)) + return &st, nil, nil + case "git": + st, ok := detectGitContext(v, "1") + if !ok { + return nil, nil, errors.Errorf("invalid git context %s", v) + } + return st, nil, nil + case "http", "https": + st, ok := detectGitContext(v, "1") + if !ok { + httpst := llb.HTTP(v, llb.WithCustomName("[context "+name+"] "+v)) + st = &httpst + } + return st, nil, nil + case "local": + st := llb.Local(vv[1], llb.WithCustomName("[context "+name+"] load from client"), llb.SessionID(c.BuildOpts().SessionID), llb.SharedKeyHint("context:"+name)) + return &st, nil, nil + case "input": + inputs, err := c.Inputs(ctx) + if err != nil { + return nil, nil, err + } + st, ok := inputs[vv[1]] + if !ok { + return nil, nil, errors.Errorf("invalid input %s for %s", vv[1], name) + } + md, ok := opts["input-metadata:"+vv[1]] + if ok { + m := make(map[string][]byte) + if err := json.Unmarshal([]byte(md), &m); err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md) } - md, ok := opts["input-metadata:"+vv[1]] + dt, ok := m["containerimage.config"] if ok { - m := make(map[string][]byte) - if err := json.Unmarshal([]byte(md), &m); err != nil { - return nil, nil, errors.Wrapf(err, "failed to parse input metadata %s", md) + st, err = st.WithImageConfig([]byte(dt)) + if err != nil { + return nil, nil, err } - dt, ok := m["containerimage.config"] - if ok { - st, err = st.WithImageConfig([]byte(dt)) - if err != nil { - return nil, nil, err - } - var img dockerfile2llb.Image - if err := json.Unmarshal(dt, &img); err != nil { - return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name) - } - return &st, &img, nil + var img dockerfile2llb.Image + if err := json.Unmarshal(dt, &img); err != nil { + return nil, nil, errors.Wrapf(err, "failed to parse image config for %s", name) } + return &st, &img, nil } - return &st, nil, nil - default: - return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name) } + return &st, nil, nil + default: + return nil, nil, errors.Errorf("unsupported context source %s for %s", vv[0], name) } } diff --git a/frontend/dockerfile/dockerfile_test.go b/frontend/dockerfile/dockerfile_test.go index fa67fa5e7df6..3aaa9f0e32c3 100644 --- a/frontend/dockerfile/dockerfile_test.go +++ b/frontend/dockerfile/dockerfile_test.go @@ -122,6 +122,7 @@ var allTests = integration.TestFuncs( testNamedImageContext, testNamedLocalContext, testNamedInputContext, + testNamedMultiplatformInputContext, ) var fileOpTests = integration.TestFuncs( @@ -5627,6 +5628,155 @@ COPY --from=build /foo /out / require.Equal(t, "foo is bar\n", string(dt)) } +func testNamedMultiplatformInputContext(t *testing.T, sb integration.Sandbox) { + ctx := sb.Context() + + c, err := client.New(ctx, sb.Address()) + require.NoError(t, err) + defer c.Close() + + dockerfile := []byte(` +FROM --platform=$BUILDPLATFORM alpine +ARG TARGETARCH +ENV FOO=bar-$TARGETARCH +RUN echo "foo $TARGETARCH" > /out +`) + + dir, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + dockerfile2 := []byte(` +FROM base AS build +RUN echo "foo is $FOO" > /foo +FROM scratch +COPY --from=build /foo /out / +`) + + dir2, err := tmpdir( + fstest.CreateFile("Dockerfile", dockerfile2, 0600), + ) + require.NoError(t, err) + defer os.RemoveAll(dir) + + f := getFrontend(t, sb) + + b := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + res, err := f.SolveGateway(ctx, c, gateway.SolveRequest{ + FrontendOpt: map[string]string{ + "platform": "linux/amd64,linux/arm64", + }, + }) + if err != nil { + return nil, err + } + + if len(res.Refs) != 2 { + return nil, errors.Errorf("expected 2 refs, got %d", len(res.Refs)) + } + + inputs := map[string]*pb.Definition{} + st, err := res.Refs["linux/amd64"].ToState() + if err != nil { + return nil, err + } + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + inputs["base::linux/amd64"] = def.ToPB() + + st, err = res.Refs["linux/arm64"].ToState() + if err != nil { + return nil, err + } + def, err = st.Marshal(ctx) + if err != nil { + return nil, err + } + inputs["base::linux/arm64"] = def.ToPB() + + frontendOpt := map[string]string{ + "dockerfilekey": builder.DefaultLocalNameDockerfile + "2", + "context:base::linux/amd64": "input:base::linux/amd64", + "context:base::linux/arm64": "input:base::linux/arm64", + "platform": "linux/amd64,linux/arm64", + } + + dt, ok := res.Metadata["containerimage.config/linux/amd64"] + if !ok { + return nil, errors.Errorf("no containerimage.config in metadata") + } + dt, err = json.Marshal(map[string][]byte{ + "containerimage.config": dt, + }) + if err != nil { + return nil, err + } + frontendOpt["input-metadata:base::linux/amd64"] = string(dt) + + dt, ok = res.Metadata["containerimage.config/linux/arm64"] + if !ok { + return nil, errors.Errorf("no containerimage.config in metadata") + } + dt, err = json.Marshal(map[string][]byte{ + "containerimage.config": dt, + }) + if err != nil { + return nil, err + } + frontendOpt["input-metadata:base::linux/arm64"] = string(dt) + + res, err = f.SolveGateway(ctx, c, gateway.SolveRequest{ + FrontendOpt: frontendOpt, + FrontendInputs: inputs, + }) + if err != nil { + return nil, err + } + return res, nil + } + + product := "buildkit_test" + + destDir, err := ioutil.TempDir("", "buildkit") + require.NoError(t, err) + defer os.RemoveAll(destDir) + + _, err = c.Build(ctx, client.SolveOpt{ + LocalDirs: map[string]string{ + builder.DefaultLocalNameDockerfile: dir, + builder.DefaultLocalNameContext: dir, + builder.DefaultLocalNameDockerfile + "2": dir2, + }, + Exports: []client.ExportEntry{ + { + Type: client.ExporterLocal, + OutputDir: destDir, + }, + }, + }, product, b, nil) + require.NoError(t, err) + + dt, err := ioutil.ReadFile(filepath.Join(destDir, "linux_amd64/out")) + require.NoError(t, err) + require.Equal(t, "foo amd64\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_amd64/foo")) + require.NoError(t, err) + require.Equal(t, "foo is bar-amd64\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_arm64/out")) + require.NoError(t, err) + require.Equal(t, "foo arm64\n", string(dt)) + + dt, err = ioutil.ReadFile(filepath.Join(destDir, "linux_arm64/foo")) + require.NoError(t, err) + require.Equal(t, "foo is bar-arm64\n", string(dt)) +} + func tmpdir(appliers ...fstest.Applier) (string, error) { tmpdir, err := ioutil.TempDir("", "buildkit-dockerfile") if err != nil { From 23709ef3163b660caef6056e19ddd31c11cc97b6 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 13 Dec 2021 23:36:53 -0800 Subject: [PATCH 3/3] dockerfile: set current platform on image based named context Signed-off-by: Tonis Tiigi --- frontend/dockerfile/builder/build.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index 18d175ce438a..5ea0eae1b797 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -814,7 +814,7 @@ func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Conte if p != nil { name := name + "::" + platforms.Format(platforms.Normalize(*p)) - st, img, err := contextByName(ctx, c, name) + st, img, err := contextByName(ctx, c, name, p) if err != nil { return nil, nil, err } @@ -822,11 +822,11 @@ func contextByNameFunc(c client.Client, p *ocispecs.Platform) func(context.Conte return st, img, nil } } - return contextByName(ctx, c, name) + return contextByName(ctx, c, name, p) } } -func contextByName(ctx context.Context, c client.Client, name string) (*llb.State, *dockerfile2llb.Image, error) { +func contextByName(ctx context.Context, c client.Client, name string, platform *ocispecs.Platform) (*llb.State, *dockerfile2llb.Image, error) { opts := c.BuildOpts().Opts v, ok := opts["context:"+name] if !ok { @@ -839,7 +839,14 @@ func contextByName(ctx context.Context, c client.Client, name string) (*llb.Stat } switch vv[0] { case "docker-image": - st := llb.Image(strings.TrimPrefix(vv[1], "//"), llb.WithCustomName("[context "+name+"] "+vv[1]), llb.WithMetaResolver(c)) + imgOpt := []llb.ImageOption{ + llb.WithCustomName("[context " + name + "] " + vv[1]), + llb.WithMetaResolver(c), + } + if platform != nil { + imgOpt = append(imgOpt, llb.Platform(*platform)) + } + st := llb.Image(strings.TrimPrefix(vv[1], "//"), imgOpt...) return &st, nil, nil case "git": st, ok := detectGitContext(v, "1")