From b2f93223e0af19284c7fa1611fa949b1df57c8ee Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 27 Jan 2025 08:26:59 +0100 Subject: [PATCH 1/6] use bake by default Signed-off-by: Nicolas De Loof --- pkg/compose/build_bake.go | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/pkg/compose/build_bake.go b/pkg/compose/build_bake.go index a8468f8be05..7f1b77bffcb 100644 --- a/pkg/compose/build_bake.go +++ b/pkg/compose/build_bake.go @@ -48,40 +48,33 @@ import ( ) func buildWithBake(dockerCli command.Cli) (bool, error) { - b, ok := os.LookupEnv("COMPOSE_BAKE") - if !ok { - if dockerCli.ConfigFile().Plugins["compose"]["build"] == "bake" { - b, ok = "true", true - } - } - if !ok { - return false, nil - } - bake, err := strconv.ParseBool(b) - if err != nil { - return false, err - } - if !bake { - return false, nil - } - enabled, err := dockerCli.BuildKitEnabled() if err != nil { return false, err } if !enabled { - logrus.Warnf("Docker Compose is configured to build using Bake, but buildkit isn't enabled") + return false, nil } _, err = manager.GetPlugin("buildx", dockerCli, &cobra.Command{}) if err != nil { if manager.IsNotFound(err) { - logrus.Warnf("Docker Compose is configured to build using Bake, but buildx isn't installed") + logrus.Warn("buildx plugin not installed.") return false, nil } return false, err } - return true, err + + b, ok := os.LookupEnv("COMPOSE_BAKE") + if !ok { + // User can opt-out with COMPOSE_BAKE=false but we use bake by default + return true, nil + } + bake, err := strconv.ParseBool(b) + if err != nil { + return false, err + } + return bake, nil } // We _could_ use bake.* types from github.com/docker/buildx but long term plan is to remove buildx as a dependency From f890c36eb6eafcfb0c320f92ff365f9e08a56cd2 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 30 Jan 2025 10:05:04 +0100 Subject: [PATCH 2/6] pass --allow for filesystem read access Signed-off-by: Nicolas De Loof --- pkg/compose/build_bake.go | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/pkg/compose/build_bake.go b/pkg/compose/build_bake.go index 7f1b77bffcb..1099b5cbcf4 100644 --- a/pkg/compose/build_bake.go +++ b/pkg/compose/build_bake.go @@ -39,6 +39,7 @@ import ( "github.com/docker/docker/api/types/versions" "github.com/docker/docker/builder/remotecontext/urlutil" "github.com/moby/buildkit/client" + "github.com/moby/buildkit/util/gitutil" "github.com/moby/buildkit/util/progress/progressui" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -138,6 +139,7 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project } var group bakeGroup var privileged bool + var read []string for serviceName, service := range serviceToBeBuild { if service.Build == nil { @@ -168,6 +170,13 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project if options.Push && service.Image != "" { outputs = append(outputs, "type=image,push=true") } + read = append(read, build.Context) + for _, path := range build.AdditionalContexts { + _, err := gitutil.ParseGitRef(path) + if !strings.Contains(path, "://") && err != nil { + read = append(read, path) + } + } cfg.Targets[serviceName] = bakeTarget{ Context: build.Context, @@ -196,11 +205,13 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project cfg.Groups["default"] = group - b, err := json.Marshal(cfg) + b, err := json.MarshalIndent(cfg, "", " ") if err != nil { return nil, err } + logrus.Debugf("bake config:\n%s", string(b)) + metadata, err := os.CreateTemp(os.TempDir(), "compose") if err != nil { return nil, err @@ -213,9 +224,15 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project args := []string{"bake", "--file", "-", "--progress", "rawjson", "--metadata-file", metadata.Name()} mustAllow := buildx.Version != "" && versions.GreaterThanOrEqualTo(buildx.Version[1:], "0.17.0") - if privileged && mustAllow { - args = append(args, "--allow", "security.insecure") + if mustAllow { + for _, path := range read { + args = append(args, "--allow", "fs.read="+path) + } + if privileged { + args = append(args, "--allow", "security.insecure") + } } + logrus.Debugf("Executing bake with args: %v", args) cmd := exec.CommandContext(ctx, buildx.Path, args...) // Remove DOCKER_CLI_PLUGIN... variable so buildx can detect it run standalone @@ -250,16 +267,15 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project eg.Go(cmd.Wait) for { decoder := json.NewDecoder(pipe) - var s client.SolveStatus - err := decoder.Decode(&s) + var status client.SolveStatus + err := decoder.Decode(&status) if err != nil { if errors.Is(err, io.EOF) { break } - // bake displays build details at the end of a build, which isn't a json SolveStatus continue } - ch <- &s + ch <- &status } close(ch) // stop build progress UI From 6a1e0df01f753a618b5e7fd06c1552d43d06ba1e Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 30 Jan 2025 14:54:51 +0100 Subject: [PATCH 3/6] fix exporter to only load image for default platform Signed-off-by: Nicolas De Loof --- pkg/compose/build_bake.go | 12 ++++++++---- pkg/e2e/build_test.go | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pkg/compose/build_bake.go b/pkg/compose/build_bake.go index 1099b5cbcf4..96e615aea59 100644 --- a/pkg/compose/build_bake.go +++ b/pkg/compose/build_bake.go @@ -166,10 +166,14 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project privileged = true } - outputs := []string{"type=docker"} - if options.Push && service.Image != "" { - outputs = append(outputs, "type=image,push=true") + var output string + push := options.Push && service.Image != "" + if len(service.Build.Platforms) > 1 { + output = fmt.Sprintf("type=image,push=%t", push) + } else { + output = fmt.Sprintf("type=docker,load=true,push=%t", push) } + read = append(read, build.Context) for _, path := range build.AdditionalContexts { _, err := gitutil.ParseGitRef(path) @@ -198,7 +202,7 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project ShmSize: build.ShmSize, Ulimits: toBakeUlimits(build.Ulimits), Entitlements: entitlements, - Outputs: outputs, + Outputs: []string{output}, } group.Targets = append(group.Targets, serviceName) } diff --git a/pkg/e2e/build_test.go b/pkg/e2e/build_test.go index 9ad1a2e192b..8ca83ba2bbb 100644 --- a/pkg/e2e/build_test.go +++ b/pkg/e2e/build_test.go @@ -305,7 +305,7 @@ func TestBuildPlatformsWithCorrectBuildxConfig(t *testing.T) { "-f", "fixtures/build-test/platforms/compose-unsupported-platform.yml", "build") res.Assert(t, icmd.Expected{ ExitCode: 1, - Err: "no match for platform in", + Err: "no match for platform", }) }) From ff2ee81c2cea9df8143c39a9b74e33cc8d2c5326 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 30 Jan 2025 11:36:25 +0100 Subject: [PATCH 4/6] capture error message reported by bake and forward to compose Signed-off-by: Nicolas De Loof --- pkg/compose/build.go | 6 ++---- pkg/compose/build_bake.go | 20 ++++++++++++++------ pkg/e2e/build_test.go | 3 +-- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/pkg/compose/build.go b/pkg/compose/build.go index af38c05edc8..7221ed0e5e5 100644 --- a/pkg/compose/build.go +++ b/pkg/compose/build.go @@ -56,10 +56,8 @@ func (s *composeService) Build(ctx context.Context, project *types.Project, opti if err != nil { return err } - return progress.RunWithTitle(ctx, func(ctx context.Context) error { - _, err := s.build(ctx, project, options, nil) - return err - }, s.stdinfo(), "Building") + _, err = s.build(ctx, project, options, nil) + return err } //nolint:gocyclo diff --git a/pkg/compose/build_bake.go b/pkg/compose/build_bake.go index 96e615aea59..dd6d2720d0e 100644 --- a/pkg/compose/build_bake.go +++ b/pkg/compose/build_bake.go @@ -17,12 +17,12 @@ package compose import ( + "bufio" "bytes" "context" "encoding/json" "errors" "fmt" - "io" "os" "os/exec" "path/filepath" @@ -264,18 +264,23 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project return nil, err } + var errMessage string + scanner := bufio.NewScanner(pipe) + scanner.Split(bufio.ScanLines) + err = cmd.Start() if err != nil { return nil, err } eg.Go(cmd.Wait) - for { - decoder := json.NewDecoder(pipe) + for scanner.Scan() { + line := scanner.Text() + decoder := json.NewDecoder(strings.NewReader(line)) var status client.SolveStatus err := decoder.Decode(&status) if err != nil { - if errors.Is(err, io.EOF) { - break + if strings.HasPrefix(line, "ERROR: ") { + errMessage = line[7:] } continue } @@ -285,7 +290,10 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project err = eg.Wait() if err != nil { - return nil, err + if errMessage != "" { + return nil, errors.New(errMessage) + } + return nil, fmt.Errorf("failed to execute bake: %w", err) } b, err = os.ReadFile(metadata.Name()) diff --git a/pkg/e2e/build_test.go b/pkg/e2e/build_test.go index 8ca83ba2bbb..3459a86168f 100644 --- a/pkg/e2e/build_test.go +++ b/pkg/e2e/build_test.go @@ -403,8 +403,7 @@ func TestBuildPlatformsStandardErrors(t *testing.T) { res := c.RunDockerComposeCmdNoCheck(t, "--project-directory", "fixtures/build-test/platforms", "build") res.Assert(t, icmd.Expected{ ExitCode: 1, - Err: `Multi-platform build is not supported for the docker driver. -Switch to a different driver, or turn on the containerd image store, and try again.`, + Err: "Multi-platform build is not supported for the docker driver.", }) }) From 6cb4273357daf759bfd382e0ffbb7d45936bcc30 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 30 Jan 2025 13:24:42 +0100 Subject: [PATCH 5/6] fix bake uses selected builder Signed-off-by: Nicolas De Loof --- pkg/compose/build_bake.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/compose/build_bake.go b/pkg/compose/build_bake.go index dd6d2720d0e..c1385d5f95b 100644 --- a/pkg/compose/build_bake.go +++ b/pkg/compose/build_bake.go @@ -236,6 +236,11 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project args = append(args, "--allow", "security.insecure") } } + + if options.Builder != "" { + args = append(args, "--builder", options.Builder) + } + logrus.Debugf("Executing bake with args: %v", args) cmd := exec.CommandContext(ctx, buildx.Path, args...) From 05e173e4e52cd976ba5130b3f381f0a3709deecf Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 30 Jan 2025 13:38:50 +0100 Subject: [PATCH 6/6] run TestBuildImageDependencies in backward compatibility mode Signed-off-by: Nicolas De Loof --- cmd/compose/build.go | 21 ++++++++++----------- pkg/api/api.go | 2 -- pkg/compose/build_bake.go | 10 ++++------ 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/cmd/compose/build.go b/cmd/compose/build.go index 4b0e59bfd81..ca9e0f25932 100644 --- a/cmd/compose/build.go +++ b/cmd/compose/build.go @@ -68,17 +68,16 @@ func (opts buildOptions) toAPIBuildOptions(services []string) (api.BuildOptions, uiMode = "rawjson" } return api.BuildOptions{ - Pull: opts.pull, - Push: opts.push, - Progress: uiMode, - Args: types.NewMappingWithEquals(opts.args), - NoCache: opts.noCache, - Quiet: opts.quiet, - Services: services, - Deps: opts.deps, - SSHs: SSHKeys, - Builder: builderName, - Compatibility: opts.Compatibility, + Pull: opts.pull, + Push: opts.push, + Progress: uiMode, + Args: types.NewMappingWithEquals(opts.args), + NoCache: opts.noCache, + Quiet: opts.quiet, + Services: services, + Deps: opts.deps, + SSHs: SSHKeys, + Builder: builderName, }, nil } diff --git a/pkg/api/api.go b/pkg/api/api.go index a5b889ce3a0..06aea8064f6 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -155,8 +155,6 @@ type BuildOptions struct { Memory int64 // Builder name passed in the command line Builder string - // Compatibility let compose run with best backward compatibility - Compatibility bool } // Apply mutates project according to build options diff --git a/pkg/compose/build_bake.go b/pkg/compose/build_bake.go index c1385d5f95b..0d6a736a14f 100644 --- a/pkg/compose/build_bake.go +++ b/pkg/compose/build_bake.go @@ -184,7 +184,7 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project cfg.Targets[serviceName] = bakeTarget{ Context: build.Context, - Contexts: additionalContexts(build.AdditionalContexts, service.DependsOn, options.Compatibility), + Contexts: additionalContexts(build.AdditionalContexts, service.DependsOn), Dockerfile: dockerFilePath(build.Context, build.Dockerfile), DockerfileInline: build.DockerfileInline, Args: args, @@ -320,12 +320,10 @@ func (s *composeService) doBuildBake(ctx context.Context, project *types.Project return results, nil } -func additionalContexts(contexts types.Mapping, dependencies types.DependsOnConfig, compatibility bool) map[string]string { +func additionalContexts(contexts types.Mapping, dependencies types.DependsOnConfig) map[string]string { ac := map[string]string{} - if compatibility { - for name := range dependencies { - ac[name] = "target:" + name - } + for name := range dependencies { + ac[name] = "target:" + name } for k, v := range contexts { ac[k] = v