From 31fe2f12a441882829ca416ecdf19271d64e4444 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Wed, 15 Sep 2021 13:30:48 +0200 Subject: [PATCH 1/2] GitHub Actions cross Signed-off-by: CrazyMax (cherry picked from commit 81b3d6bf6e9ed4a528e8fdc4e9fda5987812b095) --- .circleci/config.yml | 31 ----------------------- .github/workflows/build.yml | 50 +++++++++++++++++++++++++++++++++++++ Makefile | 12 --------- README.md | 16 ++++++------ docker-bake.hcl | 26 +++---------------- docker.Makefile | 13 ++++++++-- 6 files changed, 73 insertions(+), 75 deletions(-) create mode 100644 .github/workflows/build.yml diff --git a/.circleci/config.yml b/.circleci/config.yml index 3e3fa75cdbc9..107662091604 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,36 +2,6 @@ version: 2 jobs: - cross: - working_directory: /work - docker: [{image: 'docker:20.10-git'}] - environment: - DOCKER_BUILDKIT: 1 - BUILDX_VERSION: "v0.6.0" - parallelism: 3 - steps: - - checkout - - setup_remote_docker: - version: 20.10.6 - reusable: true - exclusive: false - - run: - name: "Docker version" - command: docker version - - run: - name: "Docker info" - command: docker info - - run: apk add make curl - - run: mkdir -vp ~/.docker/cli-plugins/ - - run: curl -fsSL --output ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64 - - run: chmod a+x ~/.docker/cli-plugins/docker-buildx - - run: docker buildx version - - run: docker context create buildctx - - run: docker buildx create --use buildctx && docker buildx inspect --bootstrap - - run: GROUP_INDEX=$CIRCLE_NODE_INDEX GROUP_TOTAL=$CIRCLE_NODE_TOTAL docker buildx bake cross --progress=plain - - store_artifacts: - path: /work/build - test: working_directory: /work docker: [{image: 'docker:20.10-git'}] @@ -112,6 +82,5 @@ workflows: version: 2 ci: jobs: - - cross - test - validate diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000000..a2f6278d04a5 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,50 @@ +name: build + +on: + workflow_dispatch: + push: + branches: + - 'master' + - '[0-9]+.[0-9]{2}' + tags: + - 'v*' + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: + - cross + - dynbinary-cross + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Run ${{ matrix.target }} + uses: docker/bake-action@v1 + with: + targets: ${{ matrix.target }} + - + name: Flatten artifacts + working-directory: ./build + run: | + for dir in */; do + base=$(basename "$dir") + echo "Creating ${base}.tar.gz ..." + tar -cvzf "${base}.tar.gz" "$dir" + rm -rf "$dir" + done + - + name: Upload artifacts + uses: actions/upload-artifact@v2 + with: + name: ${{ matrix.target }} + path: ./build/* + if-no-files-found: error diff --git a/Makefile b/Makefile index b6c351120df8..84c928bf42cc 100644 --- a/Makefile +++ b/Makefile @@ -25,18 +25,10 @@ test-coverage: ## run test coverage fmt: go list -f {{.Dir}} ./... | xargs gofmt -w -s -d -.PHONY: binary -binary: - docker buildx bake binary - .PHONY: plugins plugins: ## build example CLI plugins ./scripts/build/plugins -.PHONY: cross -cross: - docker buildx bake cross - .PHONY: plugins-windows plugins-windows: ## build example CLI plugins for Windows ./scripts/build/plugins-windows @@ -45,10 +37,6 @@ plugins-windows: ## build example CLI plugins for Windows plugins-osx: ## build example CLI plugins for macOS ./scripts/build/plugins-osx -.PHONY: dynbinary -dynbinary: ## build dynamically linked binary - USE_GLIBC=1 docker buildx bake dynbinary - vendor: vendor.conf ## check that vendor matches vendor.conf rm -rf vendor bash -c 'vndr |& grep -v -i clone | tee ./vndr.log' diff --git a/README.md b/README.md index 770d68c1db8f..9044911060a7 100644 --- a/README.md +++ b/README.md @@ -14,38 +14,38 @@ Development Build CLI from source: -``` +```console $ docker buildx bake ``` Build binaries for all supported platforms: -``` +```console $ docker buildx bake cross ``` Build for a specific platform: -``` +```console $ docker buildx bake --set binary.platform=linux/arm64 ``` Build dynamic binary for glibc or musl: -``` +```console $ USE_GLIBC=1 docker buildx bake dynbinary ``` Run all linting: -``` -$ make -f docker.Makefile lint +```console +$ docker buildx bake lint shellcheck ``` List all the available targets: -``` +```console $ make help ``` @@ -53,7 +53,7 @@ $ make help Start an interactive development environment: -``` +```console $ make -f docker.Makefile shell ``` diff --git a/docker-bake.hcl b/docker-bake.hcl index 08662a4c31c6..59598dfe69af 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -32,34 +32,16 @@ target "dynbinary" { } } -variable "GROUP_TOTAL" { - default = "1" -} - -variable "GROUP_INDEX" { - default = "0" -} - -function "platforms" { - params = [USE_GLIBC] - result = concat(["linux/amd64", "linux/386", "linux/arm64", "linux/arm", "linux/ppc64le", "linux/s390x", "darwin/amd64", "darwin/arm64", "windows/amd64", "windows/arm", "windows/386"], USE_GLIBC!=""?[]:["windows/arm64"]) -} - -function "glen" { - params = [platforms, GROUP_TOTAL] - result = ceil(length(platforms)/GROUP_TOTAL) -} - -target "_all_platforms" { - platforms = slice(platforms(USE_GLIBC), GROUP_INDEX*glen(platforms(USE_GLIBC), GROUP_TOTAL),min(length(platforms(USE_GLIBC)), (GROUP_INDEX+1)*glen(platforms(USE_GLIBC), GROUP_TOTAL))) +target "platforms" { + platforms = concat(["linux/amd64", "linux/386", "linux/arm64", "linux/arm", "linux/ppc64le", "linux/s390x", "darwin/amd64", "darwin/arm64", "windows/amd64", "windows/arm", "windows/386"], USE_GLIBC!=""?[]:["windows/arm64"]) } target "cross" { - inherits = ["binary", "_all_platforms"] + inherits = ["binary", "platforms"] } target "dynbinary-cross" { - inherits = ["dynbinary", "_all_platforms"] + inherits = ["dynbinary", "platforms"] } target "lint" { diff --git a/docker.Makefile b/docker.Makefile index 324f788d9236..5363a962a61e 100644 --- a/docker.Makefile +++ b/docker.Makefile @@ -42,8 +42,9 @@ build_e2e_image: DOCKER_RUN_NAME_OPTION := $(if $(DOCKER_CLI_CONTAINER_NAME),--name $(DOCKER_CLI_CONTAINER_NAME),) DOCKER_RUN := docker run --rm $(ENVVARS) $(DOCKER_CLI_MOUNTS) $(DOCKER_RUN_NAME_OPTION) -binary: build_binary_native_image ## build the CLI - $(DOCKER_RUN) $(BINARY_NATIVE_IMAGE_NAME) +.PHONY: binary +binary: + docker buildx bake binary build: binary ## alias for binary @@ -62,6 +63,10 @@ test-unit: build_docker_image ## run unit tests (using go test) .PHONY: test ## run unit and e2e tests test: test-unit test-e2e +.PHONY: cross +cross: + docker buildx bake cross + .PHONY: plugins-windows plugins-windows: build_cross_image ## build the example CLI plugins for Windows $(DOCKER_RUN) $(CROSS_IMAGE_NAME) make $@ @@ -70,6 +75,10 @@ plugins-windows: build_cross_image ## build the example CLI plugins for Windows plugins-osx: build_cross_image ## build the example CLI plugins for macOS $(DOCKER_RUN) $(CROSS_IMAGE_NAME) make $@ +.PHONY: dynbinary +dynbinary: ## build dynamically linked binary + USE_GLIBC=1 docker buildx bake dynbinary + .PHONY: dev dev: build_docker_image ## start a build container in interactive mode for in-container development $(DOCKER_RUN) -it \ From f0f6ec85e4e82c5088dc38e5c11bf90292414170 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Wed, 22 Sep 2021 11:51:54 +0200 Subject: [PATCH 2/2] Buildx by default Signed-off-by: CrazyMax --- cli-plugins/manager/manager.go | 30 ++ cli-plugins/manager/manager_test.go | 23 ++ cli/command/cli.go | 15 - cli/command/image/build.go | 102 +----- cli/command/image/build_buildkit.go | 525 ---------------------------- cli/command/image/build_session.go | 69 ---- cli/command/image/build_test.go | 71 ---- cmd/docker/docker.go | 84 ++++- 8 files changed, 126 insertions(+), 793 deletions(-) delete mode 100644 cli/command/image/build_buildkit.go delete mode 100644 cli/command/image/build_session.go diff --git a/cli-plugins/manager/manager.go b/cli-plugins/manager/manager.go index 50f7208ea334..89727f7cac65 100644 --- a/cli-plugins/manager/manager.go +++ b/cli-plugins/manager/manager.go @@ -104,6 +104,36 @@ func listPluginCandidates(dirs []string) (map[string][]string, error) { return result, nil } +// GetPlugin returns a plugin on the system by its name +func GetPlugin(name string, dockerCli command.Cli, rootcmd *cobra.Command) (*Plugin, error) { + pluginDirs, err := getPluginDirs(dockerCli) + if err != nil { + return nil, err + } + + candidates, err := listPluginCandidates(pluginDirs) + if err != nil { + return nil, err + } + + if paths, ok := candidates[name]; ok { + if len(paths) == 0 { + return nil, errPluginNotFound(name) + } + c := &candidate{paths[0]} + p, err := newPlugin(c, rootcmd) + if err != nil { + return nil, err + } + if !IsNotFound(p.Err) { + p.ShadowedPaths = paths[1:] + } + return &p, nil + } + + return nil, errPluginNotFound(name) +} + // ListPlugins produces a list of the plugins available on the system func ListPlugins(dockerCli command.Cli, rootcmd *cobra.Command) ([]Plugin, error) { pluginDirs, err := getPluginDirs(dockerCli) diff --git a/cli-plugins/manager/manager_test.go b/cli-plugins/manager/manager_test.go index 0da315c5857e..414e6899e70e 100644 --- a/cli-plugins/manager/manager_test.go +++ b/cli-plugins/manager/manager_test.go @@ -82,6 +82,29 @@ func TestListPluginCandidates(t *testing.T) { assert.DeepEqual(t, candidates, exp) } +func TestGetPlugin(t *testing.T) { + dir := fs.NewDir(t, t.Name(), + fs.WithFile("docker-bbb", ` +#!/bin/sh +echo '{"SchemaVersion":"0.1.0"}'`, fs.WithMode(0777)), + fs.WithFile("docker-aaa", ` +#!/bin/sh +echo '{"SchemaVersion":"0.1.0"}'`, fs.WithMode(0777)), + ) + defer dir.Remove() + + cli := test.NewFakeCli(nil) + cli.SetConfigFile(&configfile.ConfigFile{CLIPluginsExtraDirs: []string{dir.Path()}}) + + plugin, err := GetPlugin("bbb", cli, &cobra.Command{}) + assert.NilError(t, err) + assert.Equal(t, plugin.Name, "bbb") + + _, err = GetPlugin("ccc", cli, &cobra.Command{}) + assert.Error(t, err, "Error: No such CLI plugin: ccc") + assert.Assert(t, IsNotFound(err)) +} + func TestListPluginsIsSorted(t *testing.T) { dir := fs.NewDir(t, t.Name(), fs.WithFile("docker-bbb", ` diff --git a/cli/command/cli.go b/cli/command/cli.go index fe6444f42f8d..35d2d54018fb 100644 --- a/cli/command/cli.go +++ b/cli/command/cli.go @@ -7,7 +7,6 @@ import ( "os" "path/filepath" "runtime" - "strconv" "strings" "time" @@ -171,20 +170,6 @@ func (cli *DockerCli) ContentTrustEnabled() bool { return cli.contentTrust } -// BuildKitEnabled returns whether buildkit is enabled either through a daemon setting -// or otherwise the client-side DOCKER_BUILDKIT environment variable -func BuildKitEnabled(si ServerInfo) (bool, error) { - buildkitEnabled := si.BuildkitVersion == types.BuilderBuildKit - if buildkitEnv := os.Getenv("DOCKER_BUILDKIT"); buildkitEnv != "" { - var err error - buildkitEnabled, err = strconv.ParseBool(buildkitEnv) - if err != nil { - return false, errors.Wrap(err, "DOCKER_BUILDKIT environment variable expects boolean value") - } - } - return buildkitEnabled, nil -} - // ManifestStore returns a store for local manifests func (cli *DockerCli) ManifestStore() manifeststore.Store { // TODO: support override default location from config file diff --git a/cli/command/image/build.go b/cli/command/image/build.go index 64b51113c018..1640a9a7449e 100644 --- a/cli/command/image/build.go +++ b/cli/command/image/build.go @@ -5,7 +5,6 @@ import ( "bufio" "bytes" "context" - "encoding/csv" "encoding/json" "fmt" "io" @@ -57,7 +56,6 @@ type buildOptions struct { isolation string quiet bool noCache bool - progress string rm bool forceRm bool pull bool @@ -71,9 +69,6 @@ type buildOptions struct { stream bool platform string untrusted bool - secrets []string - ssh []string - outputs []string } // dockerfileFromStdin returns true when the user specified that the Dockerfile @@ -118,40 +113,26 @@ func NewBuildCommand(dockerCli command.Cli) *cobra.Command { flags.VarP(&options.tags, "tag", "t", "Name and optionally a tag in the 'name:tag' format") flags.Var(&options.buildArgs, "build-arg", "Set build-time variables") flags.Var(options.ulimits, "ulimit", "Ulimit options") - flags.SetAnnotation("ulimit", "no-buildkit", nil) flags.StringVarP(&options.dockerfileName, "file", "f", "", "Name of the Dockerfile (Default is 'PATH/Dockerfile')") flags.VarP(&options.memory, "memory", "m", "Memory limit") - flags.SetAnnotation("memory", "no-buildkit", nil) flags.Var(&options.memorySwap, "memory-swap", "Swap limit equal to memory plus swap: '-1' to enable unlimited swap") - flags.SetAnnotation("memory-swap", "no-buildkit", nil) flags.Var(&options.shmSize, "shm-size", "Size of /dev/shm") - flags.SetAnnotation("shm-size", "no-buildkit", nil) flags.Int64VarP(&options.cpuShares, "cpu-shares", "c", 0, "CPU shares (relative weight)") - flags.SetAnnotation("cpu-shares", "no-buildkit", nil) flags.Int64Var(&options.cpuPeriod, "cpu-period", 0, "Limit the CPU CFS (Completely Fair Scheduler) period") - flags.SetAnnotation("cpu-period", "no-buildkit", nil) flags.Int64Var(&options.cpuQuota, "cpu-quota", 0, "Limit the CPU CFS (Completely Fair Scheduler) quota") - flags.SetAnnotation("cpu-quota", "no-buildkit", nil) flags.StringVar(&options.cpuSetCpus, "cpuset-cpus", "", "CPUs in which to allow execution (0-3, 0,1)") - flags.SetAnnotation("cpuset-cpus", "no-buildkit", nil) flags.StringVar(&options.cpuSetMems, "cpuset-mems", "", "MEMs in which to allow execution (0-3, 0,1)") - flags.SetAnnotation("cpuset-mems", "no-buildkit", nil) flags.StringVar(&options.cgroupParent, "cgroup-parent", "", "Optional parent cgroup for the container") - flags.SetAnnotation("cgroup-parent", "no-buildkit", nil) flags.StringVar(&options.isolation, "isolation", "", "Container isolation technology") flags.Var(&options.labels, "label", "Set metadata for an image") flags.BoolVar(&options.noCache, "no-cache", false, "Do not use cache when building the image") flags.BoolVar(&options.rm, "rm", true, "Remove intermediate containers after a successful build") - flags.SetAnnotation("rm", "no-buildkit", nil) flags.BoolVar(&options.forceRm, "force-rm", false, "Always remove intermediate containers") - flags.SetAnnotation("force-rm", "no-buildkit", nil) flags.BoolVarP(&options.quiet, "quiet", "q", false, "Suppress the build output and print image ID on success") flags.BoolVar(&options.pull, "pull", false, "Always attempt to pull a newer version of the image") flags.StringSliceVar(&options.cacheFrom, "cache-from", []string{}, "Images to consider as cache sources") flags.BoolVar(&options.compress, "compress", false, "Compress the build context using gzip") - flags.SetAnnotation("compress", "no-buildkit", nil) flags.StringSliceVar(&options.securityOpt, "security-opt", []string{}, "Security options") - flags.SetAnnotation("security-opt", "no-buildkit", nil) flags.StringVar(&options.networkMode, "network", "default", "Set the networking mode for the RUN instructions during build") flags.SetAnnotation("network", "version", []string{"1.25"}) flags.Var(&options.extraHosts, "add-host", "Add a custom host-to-IP mapping (host:ip)") @@ -160,10 +141,6 @@ func NewBuildCommand(dockerCli command.Cli) *cobra.Command { command.AddTrustVerificationFlags(flags, &options.untrusted, dockerCli.ContentTrustEnabled()) - flags.StringVar(&options.platform, "platform", os.Getenv("DOCKER_DEFAULT_PLATFORM"), "Set platform if server is multi-platform capable") - flags.SetAnnotation("platform", "version", []string{"1.38"}) - flags.SetAnnotation("platform", "buildkit", nil) - flags.BoolVar(&options.squash, "squash", false, "Squash newly built layers into a single new layer") flags.SetAnnotation("squash", "experimental", nil) flags.SetAnnotation("squash", "version", []string{"1.25"}) @@ -171,21 +148,6 @@ func NewBuildCommand(dockerCli command.Cli) *cobra.Command { flags.BoolVar(&options.stream, "stream", false, "Stream attaches to server to negotiate build context") flags.MarkHidden("stream") - flags.StringVar(&options.progress, "progress", "auto", "Set type of progress output (auto, plain, tty). Use plain to show container output") - flags.SetAnnotation("progress", "buildkit", nil) - - flags.StringArrayVar(&options.secrets, "secret", []string{}, "Secret file to expose to the build (only if BuildKit enabled): id=mysecret,src=/local/secret") - flags.SetAnnotation("secret", "version", []string{"1.39"}) - flags.SetAnnotation("secret", "buildkit", nil) - - flags.StringArrayVar(&options.ssh, "ssh", []string{}, "SSH agent socket or keys to expose to the build (only if BuildKit enabled) (format: default|[=|[,]])") - flags.SetAnnotation("ssh", "version", []string{"1.39"}) - flags.SetAnnotation("ssh", "buildkit", nil) - - flags.StringArrayVarP(&options.outputs, "output", "o", []string{}, "Output destination (format: type=local,dest=path)") - flags.SetAnnotation("output", "version", []string{"1.40"}) - flags.SetAnnotation("output", "buildkit", nil) - return cmd } @@ -207,15 +169,8 @@ func (out *lastProgressOutput) WriteProgress(prog progress.Progress) error { // nolint: gocyclo func runBuild(dockerCli command.Cli, options buildOptions) error { - buildkitEnabled, err := command.BuildKitEnabled(dockerCli.ServerInfo()) - if err != nil { - return err - } - if buildkitEnabled { - return runBuildBuildKit(dockerCli, options) - } - var ( + err error buildCtx io.ReadCloser dockerfileCtx io.ReadCloser contextDir string @@ -609,58 +564,3 @@ func imageBuildOptions(dockerCli command.Cli, options buildOptions) types.ImageB Platform: options.platform, } } - -func parseOutputs(inp []string) ([]types.ImageBuildOutput, error) { - var outs []types.ImageBuildOutput - if len(inp) == 0 { - return nil, nil - } - for _, s := range inp { - csvReader := csv.NewReader(strings.NewReader(s)) - fields, err := csvReader.Read() - if err != nil { - return nil, err - } - if len(fields) == 1 && fields[0] == s && !strings.HasPrefix(s, "type=") { - if s == "-" { - outs = append(outs, types.ImageBuildOutput{ - Type: "tar", - Attrs: map[string]string{ - "dest": s, - }, - }) - } else { - outs = append(outs, types.ImageBuildOutput{ - Type: "local", - Attrs: map[string]string{ - "dest": s, - }, - }) - } - continue - } - - out := types.ImageBuildOutput{ - Attrs: map[string]string{}, - } - for _, field := range fields { - parts := strings.SplitN(field, "=", 2) - if len(parts) != 2 { - return nil, errors.Errorf("invalid value %s", field) - } - key := strings.ToLower(parts[0]) - value := parts[1] - switch key { - case "type": - out.Type = value - default: - out.Attrs[key] = value - } - } - if out.Type == "" { - return nil, errors.Errorf("type is required for output") - } - outs = append(outs, out) - } - return outs, nil -} diff --git a/cli/command/image/build_buildkit.go b/cli/command/image/build_buildkit.go deleted file mode 100644 index f65514d6e1f3..000000000000 --- a/cli/command/image/build_buildkit.go +++ /dev/null @@ -1,525 +0,0 @@ -package image - -import ( - "bytes" - "context" - "encoding/csv" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net" - "os" - "path/filepath" - "strings" - - "github.com/containerd/console" - "github.com/containerd/containerd/platforms" - "github.com/docker/cli/cli" - "github.com/docker/cli/cli/command" - "github.com/docker/cli/cli/command/image/build" - "github.com/docker/cli/opts" - "github.com/docker/docker/api/types" - "github.com/docker/docker/pkg/jsonmessage" - "github.com/docker/docker/pkg/stringid" - "github.com/docker/docker/pkg/urlutil" - controlapi "github.com/moby/buildkit/api/services/control" - "github.com/moby/buildkit/client" - "github.com/moby/buildkit/session" - "github.com/moby/buildkit/session/auth/authprovider" - "github.com/moby/buildkit/session/filesync" - "github.com/moby/buildkit/session/secrets/secretsprovider" - "github.com/moby/buildkit/session/sshforward/sshprovider" - "github.com/moby/buildkit/util/appcontext" - "github.com/moby/buildkit/util/gitutil" - "github.com/moby/buildkit/util/progress/progressui" - "github.com/moby/buildkit/util/progress/progresswriter" - "github.com/pkg/errors" - fsutiltypes "github.com/tonistiigi/fsutil/types" - "github.com/tonistiigi/go-rosetta" - "golang.org/x/sync/errgroup" -) - -const uploadRequestRemote = "upload-request" - -var errDockerfileConflict = errors.New("ambiguous Dockerfile source: both stdin and flag correspond to Dockerfiles") - -//nolint: gocyclo -func runBuildBuildKit(dockerCli command.Cli, options buildOptions) error { - ctx := appcontext.Context() - - s, err := trySession(dockerCli, options.context, false) - if err != nil { - return err - } - if s == nil { - return errors.Errorf("buildkit not supported by daemon") - } - - if options.imageIDFile != "" { - // Avoid leaving a stale file if we eventually fail - if err := os.Remove(options.imageIDFile); err != nil && !os.IsNotExist(err) { - return errors.Wrap(err, "removing image ID file") - } - } - - var ( - remote string - body io.Reader - dockerfileName = options.dockerfileName - dockerfileReader io.ReadCloser - dockerfileDir string - contextDir string - ) - - stdoutUsed := false - - switch { - case options.contextFromStdin(): - if options.dockerfileFromStdin() { - return errStdinConflict - } - rc, isArchive, err := build.DetectArchiveReader(dockerCli.In()) - if err != nil { - return err - } - if isArchive { - body = rc - remote = uploadRequestRemote - } else { - if options.dockerfileName != "" { - return errDockerfileConflict - } - dockerfileReader = rc - remote = clientSessionRemote - // TODO: make fssync handle empty contextdir - contextDir, _ = ioutil.TempDir("", "empty-dir") - defer os.RemoveAll(contextDir) - } - case isLocalDir(options.context): - contextDir = options.context - if options.dockerfileFromStdin() { - dockerfileReader = dockerCli.In() - } else if options.dockerfileName != "" { - dockerfileName = filepath.Base(options.dockerfileName) - dockerfileDir = filepath.Dir(options.dockerfileName) - } else { - dockerfileDir = options.context - } - remote = clientSessionRemote - case urlutil.IsGitURL(options.context): - remote = options.context - case urlutil.IsURL(options.context): - remote = options.context - default: - return errors.Errorf("unable to prepare context: path %q not found", options.context) - } - - if dockerfileReader != nil { - dockerfileName = build.DefaultDockerfileName - dockerfileDir, err = build.WriteTempDockerfile(dockerfileReader) - if err != nil { - return err - } - defer os.RemoveAll(dockerfileDir) - } - - outputs, err := parseOutputs(options.outputs) - if err != nil { - return errors.Wrapf(err, "failed to parse outputs") - } - - for _, out := range outputs { - switch out.Type { - case "local": - // dest is handled on client side for local exporter - outDir, ok := out.Attrs["dest"] - if !ok { - return errors.Errorf("dest is required for local output") - } - delete(out.Attrs, "dest") - s.Allow(filesync.NewFSSyncTargetDir(outDir)) - case "tar": - // dest is handled on client side for tar exporter - outFile, ok := out.Attrs["dest"] - if !ok { - return errors.Errorf("dest is required for tar output") - } - var w io.WriteCloser - if outFile == "-" { - if _, err := console.ConsoleFromFile(os.Stdout); err == nil { - return errors.Errorf("refusing to write output to console") - } - w = os.Stdout - stdoutUsed = true - } else { - f, err := os.Create(outFile) - if err != nil { - return errors.Wrapf(err, "failed to open %s", outFile) - } - w = f - } - output := func(map[string]string) (io.WriteCloser, error) { return w, nil } - s.Allow(filesync.NewFSSyncTarget(output)) - } - } - - if dockerfileDir != "" { - s.Allow(filesync.NewFSSyncProvider([]filesync.SyncedDir{ - { - Name: "context", - Dir: contextDir, - Map: resetUIDAndGID, - }, - { - Name: "dockerfile", - Dir: dockerfileDir, - }, - })) - } - - dockerAuthProvider := authprovider.NewDockerAuthProvider(os.Stderr) - s.Allow(dockerAuthProvider) - if len(options.secrets) > 0 { - sp, err := parseSecretSpecs(options.secrets) - if err != nil { - return errors.Wrapf(err, "could not parse secrets: %v", options.secrets) - } - s.Allow(sp) - } - - sshSpecs := options.ssh - if len(sshSpecs) == 0 && isGitSSH(remote) { - sshSpecs = []string{"default"} - } - if len(sshSpecs) > 0 { - sshp, err := parseSSHSpecs(sshSpecs) - if err != nil { - return errors.Wrapf(err, "could not parse ssh: %v", sshSpecs) - } - s.Allow(sshp) - } - - eg, ctx := errgroup.WithContext(ctx) - - dialSession := func(ctx context.Context, proto string, meta map[string][]string) (net.Conn, error) { - return dockerCli.Client().DialHijack(ctx, "/session", proto, meta) - } - eg.Go(func() error { - return s.Run(context.TODO(), dialSession) - }) - - buildID := stringid.GenerateRandomID() - if body != nil { - eg.Go(func() error { - buildOptions := types.ImageBuildOptions{ - Version: types.BuilderBuildKit, - BuildID: uploadRequestRemote + ":" + buildID, - } - - response, err := dockerCli.Client().ImageBuild(context.Background(), body, buildOptions) - if err != nil { - return err - } - defer response.Body.Close() - return nil - }) - } - - if v := os.Getenv("BUILDKIT_PROGRESS"); v != "" && options.progress == "auto" { - options.progress = v - } - - if strings.EqualFold(options.platform, "local") { - p := platforms.DefaultSpec() - p.Architecture = rosetta.NativeArch() // current binary architecture might be emulated - options.platform = platforms.Format(p) - } - - eg.Go(func() error { - defer func() { // make sure the Status ends cleanly on build errors - s.Close() - }() - - buildOptions := imageBuildOptions(dockerCli, options) - buildOptions.Version = types.BuilderBuildKit - buildOptions.Dockerfile = dockerfileName - // buildOptions.AuthConfigs = authConfigs // handled by session - buildOptions.RemoteContext = remote - buildOptions.SessionID = s.ID() - buildOptions.BuildID = buildID - buildOptions.Outputs = outputs - return doBuild(ctx, eg, dockerCli, stdoutUsed, options, buildOptions, dockerAuthProvider) - }) - - return eg.Wait() -} - -//nolint: gocyclo -func doBuild(ctx context.Context, eg *errgroup.Group, dockerCli command.Cli, stdoutUsed bool, options buildOptions, buildOptions types.ImageBuildOptions, at session.Attachable) (finalErr error) { - response, err := dockerCli.Client().ImageBuild(context.Background(), nil, buildOptions) - if err != nil { - return err - } - defer response.Body.Close() - - done := make(chan struct{}) - defer close(done) - eg.Go(func() error { - select { - case <-ctx.Done(): - return dockerCli.Client().BuildCancel(context.TODO(), buildOptions.BuildID) - case <-done: - } - return nil - }) - - t := newTracer() - ssArr := []*client.SolveStatus{} - - if err := opts.ValidateProgressOutput(options.progress); err != nil { - return err - } - - displayStatus := func(out *os.File, displayCh chan *client.SolveStatus) { - var c console.Console - // TODO: Handle tty output in non-tty environment. - if cons, err := console.ConsoleFromFile(out); err == nil && (options.progress == "auto" || options.progress == "tty") { - c = cons - } - // not using shared context to not disrupt display but let it finish reporting errors - eg.Go(func() error { - return progressui.DisplaySolveStatus(context.TODO(), "", c, out, displayCh) - }) - if s, ok := at.(interface { - SetLogger(progresswriter.Logger) - }); ok { - s.SetLogger(func(s *client.SolveStatus) { - displayCh <- s - }) - } - } - - if options.quiet { - eg.Go(func() error { - // TODO: make sure t.displayCh closes - for ss := range t.displayCh { - ssArr = append(ssArr, ss) - } - <-done - // TODO: verify that finalErr is indeed set when error occurs - if finalErr != nil { - displayCh := make(chan *client.SolveStatus) - go func() { - for _, ss := range ssArr { - displayCh <- ss - } - close(displayCh) - }() - displayStatus(os.Stderr, displayCh) - } - return nil - }) - } else { - displayStatus(os.Stderr, t.displayCh) - } - defer close(t.displayCh) - - buf := bytes.NewBuffer(nil) - - imageID := "" - writeAux := func(msg jsonmessage.JSONMessage) { - if msg.ID == "moby.image.id" { - var result types.BuildResult - if err := json.Unmarshal(*msg.Aux, &result); err != nil { - fmt.Fprintf(dockerCli.Err(), "failed to parse aux message: %v", err) - } - imageID = result.ID - return - } - t.write(msg) - } - - err = jsonmessage.DisplayJSONMessagesStream(response.Body, buf, dockerCli.Out().FD(), dockerCli.Out().IsTerminal(), writeAux) - if err != nil { - if jerr, ok := err.(*jsonmessage.JSONError); ok { - // If no error code is set, default to 1 - if jerr.Code == 0 { - jerr.Code = 1 - } - return cli.StatusError{Status: jerr.Message, StatusCode: jerr.Code} - } - } - - // Everything worked so if -q was provided the output from the daemon - // should be just the image ID and we'll print that to stdout. - // - // TODO: we may want to use Aux messages with ID "moby.image.id" regardless of options.quiet (i.e. don't send HTTP param q=1) - // instead of assuming that output is image ID if options.quiet. - if options.quiet && !stdoutUsed { - imageID = buf.String() - fmt.Fprint(dockerCli.Out(), imageID) - } - - if options.imageIDFile != "" { - if imageID == "" { - return errors.Errorf("cannot write %s because server did not provide an image ID", options.imageIDFile) - } - imageID = strings.TrimSpace(imageID) - if err := ioutil.WriteFile(options.imageIDFile, []byte(imageID), 0666); err != nil { - return errors.Wrap(err, "cannot write image ID file") - } - } - return err -} - -func resetUIDAndGID(_ string, s *fsutiltypes.Stat) bool { - s.Uid = 0 - s.Gid = 0 - return true -} - -type tracer struct { - displayCh chan *client.SolveStatus -} - -func newTracer() *tracer { - return &tracer{ - displayCh: make(chan *client.SolveStatus), - } -} - -func (t *tracer) write(msg jsonmessage.JSONMessage) { - var resp controlapi.StatusResponse - - if msg.ID != "moby.buildkit.trace" { - return - } - - var dt []byte - // ignoring all messages that are not understood - if err := json.Unmarshal(*msg.Aux, &dt); err != nil { - return - } - if err := (&resp).Unmarshal(dt); err != nil { - return - } - - s := client.SolveStatus{} - for _, v := range resp.Vertexes { - s.Vertexes = append(s.Vertexes, &client.Vertex{ - Digest: v.Digest, - Inputs: v.Inputs, - Name: v.Name, - Started: v.Started, - Completed: v.Completed, - Error: v.Error, - Cached: v.Cached, - }) - } - for _, v := range resp.Statuses { - s.Statuses = append(s.Statuses, &client.VertexStatus{ - ID: v.ID, - Vertex: v.Vertex, - Name: v.Name, - Total: v.Total, - Current: v.Current, - Timestamp: v.Timestamp, - Started: v.Started, - Completed: v.Completed, - }) - } - for _, v := range resp.Logs { - s.Logs = append(s.Logs, &client.VertexLog{ - Vertex: v.Vertex, - Stream: int(v.Stream), - Data: v.Msg, - Timestamp: v.Timestamp, - }) - } - - t.displayCh <- &s -} - -func parseSecretSpecs(sl []string) (session.Attachable, error) { - fs := make([]secretsprovider.Source, 0, len(sl)) - for _, v := range sl { - s, err := parseSecret(v) - if err != nil { - return nil, err - } - fs = append(fs, *s) - } - store, err := secretsprovider.NewStore(fs) - if err != nil { - return nil, err - } - return secretsprovider.NewSecretProvider(store), nil -} - -func parseSecret(value string) (*secretsprovider.Source, error) { - csvReader := csv.NewReader(strings.NewReader(value)) - fields, err := csvReader.Read() - if err != nil { - return nil, errors.Wrap(err, "failed to parse csv secret") - } - - fs := secretsprovider.Source{} - - var typ string - for _, field := range fields { - parts := strings.SplitN(field, "=", 2) - key := strings.ToLower(parts[0]) - - if len(parts) != 2 { - return nil, errors.Errorf("invalid field '%s' must be a key=value pair", field) - } - - value := parts[1] - switch key { - case "type": - if value != "file" && value != "env" { - return nil, errors.Errorf("unsupported secret type %q", value) - } - typ = value - case "id": - fs.ID = value - case "source", "src": - fs.FilePath = value - case "env": - fs.Env = value - default: - return nil, errors.Errorf("unexpected key '%s' in '%s'", key, field) - } - } - if typ == "env" && fs.Env == "" { - fs.Env = fs.FilePath - fs.FilePath = "" - } - return &fs, nil -} - -func parseSSHSpecs(sl []string) (session.Attachable, error) { - configs := make([]sshprovider.AgentConfig, 0, len(sl)) - for _, v := range sl { - c := parseSSH(v) - configs = append(configs, *c) - } - return sshprovider.NewSSHAgentProvider(configs) -} - -func parseSSH(value string) *sshprovider.AgentConfig { - parts := strings.SplitN(value, "=", 2) - cfg := sshprovider.AgentConfig{ - ID: parts[0], - } - if len(parts) > 1 { - cfg.Paths = strings.Split(parts[1], ",") - } - return &cfg -} - -func isGitSSH(url string) bool { - _, gitProtocol := gitutil.ParseProtocol(url) - return gitProtocol == gitutil.SSHProtocol -} diff --git a/cli/command/image/build_session.go b/cli/command/image/build_session.go deleted file mode 100644 index 0f15f51fb456..000000000000 --- a/cli/command/image/build_session.go +++ /dev/null @@ -1,69 +0,0 @@ -package image - -import ( - "context" - "crypto/rand" - "crypto/sha256" - "encoding/hex" - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/docker/cli/cli/command" - cliconfig "github.com/docker/cli/cli/config" - "github.com/docker/docker/api/types/versions" - "github.com/moby/buildkit/session" - "github.com/pkg/errors" -) - -const clientSessionRemote = "client-session" - -func isSessionSupported(dockerCli command.Cli, forStream bool) bool { - if !forStream && versions.GreaterThanOrEqualTo(dockerCli.Client().ClientVersion(), "1.39") { - return true - } - return dockerCli.ServerInfo().HasExperimental && versions.GreaterThanOrEqualTo(dockerCli.Client().ClientVersion(), "1.31") -} - -func trySession(dockerCli command.Cli, contextDir string, forStream bool) (*session.Session, error) { - if !isSessionSupported(dockerCli, forStream) { - return nil, nil - } - sharedKey := getBuildSharedKey(contextDir) - s, err := session.NewSession(context.Background(), filepath.Base(contextDir), sharedKey) - if err != nil { - return nil, errors.Wrap(err, "failed to create session") - } - return s, nil -} - -func getBuildSharedKey(dir string) string { - // build session is hash of build dir with node based randomness - s := sha256.Sum256([]byte(fmt.Sprintf("%s:%s", tryNodeIdentifier(), dir))) - return hex.EncodeToString(s[:]) -} - -func tryNodeIdentifier() string { - out := cliconfig.Dir() // return config dir as default on permission error - if err := os.MkdirAll(cliconfig.Dir(), 0700); err == nil { - sessionFile := filepath.Join(cliconfig.Dir(), ".buildNodeID") - if _, err := os.Lstat(sessionFile); err != nil { - if os.IsNotExist(err) { // create a new file with stored randomness - b := make([]byte, 32) - if _, err := rand.Read(b); err != nil { - return out - } - if err := ioutil.WriteFile(sessionFile, []byte(hex.EncodeToString(b)), 0600); err != nil { - return out - } - } - } - - dt, err := ioutil.ReadFile(sessionFile) - if err == nil { - return string(dt) - } - } - return out -} diff --git a/cli/command/image/build_test.go b/cli/command/image/build_test.go index 4317a74941fa..5c9d814c767c 100644 --- a/cli/command/image/build_test.go +++ b/cli/command/image/build_test.go @@ -5,7 +5,6 @@ import ( "bytes" "compress/gzip" "context" - "fmt" "io" "io/ioutil" "os" @@ -18,7 +17,6 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/pkg/archive" "github.com/google/go-cmp/cmp" - "github.com/moby/buildkit/session/secrets/secretsprovider" "gotest.tools/v3/assert" "gotest.tools/v3/env" "gotest.tools/v3/fs" @@ -182,75 +180,6 @@ RUN echo hello world assert.DeepEqual(t, fakeBuild.filenames(t), []string{"Dockerfile"}) } -func TestParseSecret(t *testing.T) { - type testcase struct { - value string - errExpected bool - errMatch string - source *secretsprovider.Source - } - var testcases = []testcase{ - { - value: "", - errExpected: true, - }, { - value: "foobar", - errExpected: true, - errMatch: "must be a key=value pair", - }, { - value: "foo,bar", - errExpected: true, - errMatch: "must be a key=value pair", - }, { - value: "foo=bar", - errExpected: true, - errMatch: "unexpected key", - }, { - value: "src=somefile", - source: &secretsprovider.Source{FilePath: "somefile"}, - }, { - value: "source=somefile", - source: &secretsprovider.Source{FilePath: "somefile"}, - }, { - value: "id=mysecret", - source: &secretsprovider.Source{ID: "mysecret"}, - }, { - value: "id=mysecret,src=somefile", - source: &secretsprovider.Source{ID: "mysecret", FilePath: "somefile"}, - }, { - value: "id=mysecret,source=somefile,type=file", - source: &secretsprovider.Source{ID: "mysecret", FilePath: "somefile"}, - }, { - value: "id=mysecret,src=somefile,src=othersecretfile", - source: &secretsprovider.Source{ID: "mysecret", FilePath: "othersecretfile"}, - }, { - value: "id=mysecret,src=somefile,env=SECRET", - source: &secretsprovider.Source{ID: "mysecret", FilePath: "somefile", Env: "SECRET"}, - }, { - value: "type=file", - source: &secretsprovider.Source{}, - }, { - value: "type=env", - source: &secretsprovider.Source{}, - }, { - value: "type=invalid", - errExpected: true, - errMatch: "unsupported secret type", - }, - } - - for _, tc := range testcases { - t.Run(tc.value, func(t *testing.T) { - secret, err := parseSecret(tc.value) - assert.Equal(t, err != nil, tc.errExpected, fmt.Sprintf("err=%v errExpected=%t", err, tc.errExpected)) - if tc.errMatch != "" { - assert.ErrorContains(t, err, tc.errMatch) - } - assert.DeepEqual(t, secret, tc.source) - }) - } -} - type fakeBuild struct { context *tar.Reader options types.ImageBuildOptions diff --git a/cmd/docker/docker.go b/cmd/docker/docker.go index 90945f0c478d..04bb4c03306e 100644 --- a/cmd/docker/docker.go +++ b/cmd/docker/docker.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "os/exec" + "strconv" "strings" "syscall" @@ -23,7 +24,7 @@ import ( ) var allowedAliases = map[string]struct{}{ - "builder": {}, + "builder": {}, // TODO: builder alias should be deprecated (buildx by default) } func newDockerCommand(dockerCli *command.DockerCli) *cli.TopLevelCommand { @@ -234,20 +235,80 @@ func processAliases(dockerCli command.Cli, cmd *cobra.Command, args, osArgs []st aliases = append(aliases, [2][]string{{k}, {v}}) } - if v, ok := aliasMap["builder"]; ok { - aliases = append(aliases, - [2][]string{{"build"}, {v, "build"}}, - [2][]string{{"image", "build"}, {v, "build"}}, - ) + for _, al := range aliases { + var didChange bool + args, didChange = command.StringSliceReplaceAt(args, al[0], al[1], 0) + if didChange { + osArgs, _ = command.StringSliceReplaceAt(osArgs, al[0], al[1], -1) + break + } + } + + return args, osArgs, nil +} + +func processBuildx(dockerCli command.Cli, cmd *cobra.Command, args, osArgs []string) ([]string, []string, error) { + // check DOCKER_BUILDKIT env var is present and + // if not assume we want to use buildx + if v, ok := os.LookupEnv("DOCKER_BUILDKIT"); ok { + enabled, err := strconv.ParseBool(v) + if err != nil { + return args, osArgs, errors.Wrap(err, "DOCKER_BUILDKIT environment variable expects boolean value") + } + if !enabled { + return args, osArgs, nil + } + } + + // wcow build command must use the legacy builder + if dockerCli.ServerInfo().OSType == "windows" { + return args, osArgs, nil } + + // buildx aliases + aliases := [][2][]string{ + { + {"builder", "prune"}, + {"buildx", "prune"}, + }, + { + {"build"}, + {"buildx", "build"}, + }, + { + {"image", "build"}, + {"buildx", "build"}, + }, + } + + // are we using a cmd that should be forwarded to buildx? + var forwarded bool for _, al := range aliases { var didChange bool args, didChange = command.StringSliceReplaceAt(args, al[0], al[1], 0) if didChange { + forwarded = true osArgs, _ = command.StringSliceReplaceAt(osArgs, al[0], al[1], -1) break } } + if !forwarded { + return args, osArgs, nil + } + + // check plugin is available if cmd forwarded + plugin, perr := pluginmanager.GetPlugin("buildx", dockerCli, cmd.Root()) + if perr == nil && plugin != nil { + perr = plugin.Err + } + if perr != nil { + return args, osArgs, errors.Errorf(`%v + +ERROR: Missing or broken buildx plugin. To install buildx, see + https://docs.docker.com/buildx/working-with-buildx/#install. You can + also fallback to the legacy builder with: DOCKER_BUILDKIT=0 +`, perr) + } return args, osArgs, nil } @@ -264,6 +325,11 @@ func runDocker(dockerCli *command.DockerCli) error { return err } + args, os.Args, err = processBuildx(dockerCli, cmd, args, os.Args) + if err != nil { + return err + } + args, os.Args, err = processAliases(dockerCli, cmd, args, os.Args) if err != nil { return err @@ -346,8 +412,6 @@ func hideSubcommandIf(subcmd *cobra.Command, condition func(string) bool, annota func hideUnsupportedFeatures(cmd *cobra.Command, details versionDetails) error { var ( - buildKitDisabled = func(_ string) bool { v, _ := command.BuildKitEnabled(details.ServerInfo()); return !v } - buildKitEnabled = func(_ string) bool { v, _ := command.BuildKitEnabled(details.ServerInfo()); return v } notExperimental = func(_ string) bool { return !details.ServerInfo().HasExperimental } notOSType = func(v string) bool { return v != details.ServerInfo().OSType } versionOlderThan = func(v string) bool { return versions.LessThan(details.Client().ClientVersion(), v) } @@ -365,16 +429,12 @@ func hideUnsupportedFeatures(cmd *cobra.Command, details versionDetails) error { } } - hideFlagIf(f, buildKitDisabled, "buildkit") - hideFlagIf(f, buildKitEnabled, "no-buildkit") hideFlagIf(f, notExperimental, "experimental") hideFlagIf(f, notOSType, "ostype") hideFlagIf(f, versionOlderThan, "version") }) for _, subcmd := range cmd.Commands() { - hideSubcommandIf(subcmd, buildKitDisabled, "buildkit") - hideSubcommandIf(subcmd, buildKitEnabled, "no-buildkit") hideSubcommandIf(subcmd, notExperimental, "experimental") hideSubcommandIf(subcmd, notOSType, "ostype") hideSubcommandIf(subcmd, versionOlderThan, "version")