tests: share single docker between docker-container backends

.github/workflows/build.yml

@@ -102,7 +102,7 @@ jobs:
           ./hack/test
         env:
           TEST_DOCKERD: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && '1' || '0' }}"
-          TESTFLAGS: "${{ (matrix.worker == 'docker' || matrix.worker == 'docker-container') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
+          TESTFLAGS: "${{ (matrix.worker == 'docker') && env.TESTFLAGS_DOCKER || env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$"
           TESTPKGS: "${{ matrix.pkg }}"
           SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}"
       -

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/hashicorp/go-cty-funcs v0.0.0-20200930094925-2721b1e36840
 	github.com/hashicorp/hcl/v2 v2.8.2
-	github.com/moby/buildkit v0.11.0-rc3.0.20230609092854-67a08623b95a
+	github.com/moby/buildkit v0.11.0-rc3.0.20230620112432-2d91ddcceedc
 	github.com/moby/sys/mountinfo v0.6.2
 	github.com/moby/sys/signal v0.7.0
 	github.com/morikuni/aec v1.0.0

go.sum

@@ -371,8 +371,8 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 h1:DpOJ2HYzC
 github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
-github.com/moby/buildkit v0.11.0-rc3.0.20230609092854-67a08623b95a h1:1k3bAXwxC2N1FncWijq/43sLj2OVIZ11FT0APIXWhMg=
-github.com/moby/buildkit v0.11.0-rc3.0.20230609092854-67a08623b95a/go.mod h1:4sM7BBBqXOQ+vV6LrVAOAMhZI9cVNYV5RhZCl906a64=
+github.com/moby/buildkit v0.11.0-rc3.0.20230620112432-2d91ddcceedc h1:79EnLqFEkPn6sTBXDHE546AHmYeb/QzXgGyIpli8w34=
+github.com/moby/buildkit v0.11.0-rc3.0.20230620112432-2d91ddcceedc/go.mod h1:6Y1HYDrxg3sY5gBY2FVaEvQpswBj3g/ck7aKYCjOkk0=
 github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg=
 github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo=

tests/integration.go

@@ -47,6 +47,7 @@ func buildxCmd(sb integration.Sandbox, opts ...cmdOpt) *exec.Cmd {
 
 	if builder := sb.Address(); builder != "" {
 		cmd.Args = append(cmd.Args, "--builder="+builder)
+		cmd.Env = append(cmd.Env, "BUILDX_CONFIG=/tmp/buildx-"+builder)
 	}
 	if context := sb.DockerAddress(); context != "" {
 		cmd.Env = append(cmd.Env, "DOCKER_CONTEXT="+context)

tests/workers/docker-container.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"os"
 	"os/exec"
+	"sync"
 
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/util/testutil/integration"
@@ -18,6 +19,11 @@ func InitDockerContainerWorker() {
 
 type containerWorker struct {
 	id string
+
+	docker      integration.Backend
+	dockerClose func() error
+	dockerErr   error
+	dockerOnce  sync.Once
 }
 
 func (w *containerWorker) Name() string {
@@ -29,9 +35,11 @@ func (w *containerWorker) Rootless() bool {
 }
 
 func (w *containerWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) {
-	bk, bkclose, err := dockerWorker{id: w.id}.New(ctx, cfg)
-	if err != nil {
-		return bk, bkclose, err
+	w.dockerOnce.Do(func() {
+		w.docker, w.dockerClose, w.dockerErr = dockerWorker{id: w.id}.New(ctx, cfg)
+	})
+	if w.dockerErr != nil {
+		return w.docker, w.dockerClose, w.dockerErr
 	}
 
 	name := "integration-container-" + identity.NewID()
@@ -42,25 +50,36 @@ func (w *containerWorker) New(ctx context.Context, cfg *integration.BackendConfi
 		"--driver=docker-container",
 		"--driver-opt=network=host",
 	)
-	cmd.Env = append(os.Environ(), "DOCKER_CONTEXT="+bk.DockerAddress())
+	cmd.Env = append(
+		os.Environ(),
+		"BUILDX_CONFIG=/tmp/buildx-"+name,
+		"DOCKER_CONTEXT="+w.docker.DockerAddress(),
+	)
 	if err := cmd.Run(); err != nil {
 		return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name)
 	}
 
 	cl := func() error {
-		var err error
-		if err1 := bkclose(); err == nil {
-			err = err1
-		}
 		cmd := exec.Command("buildx", "rm", "-f", name)
-		if err1 := cmd.Run(); err == nil {
-			err = err1
-		}
-		return err
+		return cmd.Run()
 	}
 
 	return &backend{
-		context: bk.DockerAddress(),
+		context: w.docker.DockerAddress(),
 		builder: name,
 	}, cl, nil
 }
+
+func (w *containerWorker) Close() error {
+	if close := w.dockerClose; close != nil {
+		return close()
+	}
+
+	// reset the worker to be ready to go again
+	w.docker = nil
+	w.dockerClose = nil
+	w.dockerErr = nil
+	w.dockerOnce = sync.Once{}
+
+	return nil
+}

tests/workers/docker.go

@@ -2,6 +2,7 @@ package workers
 
 import (
 	"context"
+	"os"
 	"os/exec"
 
 	"github.com/moby/buildkit/identity"
@@ -41,8 +42,9 @@ func (c dockerWorker) New(ctx context.Context, cfg *integration.BackendConfig) (
 		name,
 		"--docker", "host="+bk.DockerAddress(),
 	)
+	cmd.Env = append(os.Environ(), "BUILDX_CONFIG=/tmp/buildx-"+name)
 	if err := cmd.Run(); err != nil {
-		return nil, cl, errors.Wrapf(err, "failed to create buildx instance %s", name)
+		return bk, cl, errors.Wrapf(err, "failed to create buildx instance %s", name)
 	}
 
 	cl = func() error {
@@ -62,3 +64,7 @@ func (c dockerWorker) New(ctx context.Context, cfg *integration.BackendConfig) (
 		context: name,
 	}, cl, nil
 }
+
+func (c dockerWorker) Close() error {
+	return nil
+}

tests/workers/remote.go

@@ -2,6 +2,7 @@ package workers
 
 import (
 	"context"
+	"os"
 	"os/exec"
 
 	"github.com/moby/buildkit/identity"
@@ -41,6 +42,7 @@ func (w remoteWorker) New(ctx context.Context, cfg *integration.BackendConfig) (
 		"--driver=remote",
 		bk.Address(),
 	)
+	cmd.Env = append(os.Environ(), "BUILDX_CONFIG=/tmp/buildx-"+name)
 	if err := cmd.Run(); err != nil {
 		return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name)
 	}
@@ -61,3 +63,7 @@ func (w remoteWorker) New(ctx context.Context, cfg *integration.BackendConfig) (
 		builder: name,
 	}, cl, nil
 }
+
+func (w remoteWorker) Close() error {
+	return nil
+}