From e4c0cd6b413f05a5cedd3bbbe7326869acd89971 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 9 Feb 2023 08:22:45 +0100 Subject: [PATCH 1/2] integration: set custom flags for dockerd worker Signed-off-by: CrazyMax --- .github/workflows/dockerd.yml | 3 +++ hack/test | 4 ++-- util/testutil/integration/dockerd.go | 12 ++++++++---- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/dockerd.yml b/.github/workflows/dockerd.yml index 436288dc0846..2ac90c6aef75 100644 --- a/.github/workflows/dockerd.yml +++ b/.github/workflows/dockerd.yml @@ -137,3 +137,6 @@ jobs: TESTFLAGS: "${{ env.TESTFLAGS }} --run=//worker=${{ matrix.worker }}$" SKIP_INTEGRATION_TESTS: "${{ matrix.skip-integration-tests }}" CACHE_FROM: "type=gha,scope=${{ env.CACHE_GHA_SCOPE_IT }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }}" + BUILDKIT_INTEGRATION_DOCKERD_FLAGS: | + --bip=10.66.66.1/24 + --default-address-pool=base=10.66.66.0/16,size=24 diff --git a/hack/test b/hack/test index 2357f93571d0..71a5d8472b01 100755 --- a/hack/test +++ b/hack/test @@ -78,7 +78,7 @@ if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then fi if [ "$TEST_INTEGRATION" == 1 ]; then - cid=$(docker create --rm -v /tmp $coverageVol --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...}) + cid=$(docker create --rm -v /tmp $coverageVol --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...}) if [ "$TEST_DOCKERD" = "1" ]; then docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd fi @@ -118,7 +118,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then if [ -s $tarout ]; then if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$GITHUB_ACTIONS" = "true" ]; then - cid=$(docker create -v /tmp $coverageVol --rm --privileged --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test $coverageFlags --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile) + cid=$(docker create -v /tmp $coverageVol --rm --privileged --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test $coverageFlags --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile) docker cp $tarout $cid:/$release.tar if [ "$TEST_DOCKERD" = "1" ]; then docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd diff --git a/util/testutil/integration/dockerd.go b/util/testutil/integration/dockerd.go index b56390ec9598..c4083cf9bd21 100644 --- a/util/testutil/integration/dockerd.go +++ b/util/testutil/integration/dockerd.go @@ -8,6 +8,7 @@ import ( "net" "os" "path/filepath" + "strings" "time" "github.com/docker/docker/client" @@ -125,13 +126,16 @@ func (c moby) New(ctx context.Context, cfg *BackendConfig) (b Backend, cl func() return nil, nil, err } - err = d.StartWithError(cfg.Logs, + dockerdFlags := []string{ "--config-file", dockerdConfigFile, "--userland-proxy=false", - "--bip", "10.66.66.1/24", - "--default-address-pool", "base=10.66.66.0/16,size=24", "--debug", - ) + } + if s := os.Getenv("BUILDKIT_INTEGRATION_DOCKERD_FLAGS"); s != "" { + dockerdFlags = append(dockerdFlags, strings.Split(strings.TrimSpace(s), "\n")...) + } + + err = d.StartWithError(cfg.Logs, dockerdFlags...) if err != nil { return nil, nil, err } From 34880cce3f979deb0a2dab36ac0373e79a1e84b2 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 9 Feb 2023 16:42:33 +0100 Subject: [PATCH 2/2] integration: log dockerd cmd Signed-off-by: CrazyMax --- util/testutil/dockerd/daemon.go | 2 ++ util/testutil/integration/dockerd.go | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/util/testutil/dockerd/daemon.go b/util/testutil/dockerd/daemon.go index 1054e1f36946..24e05db02521 100644 --- a/util/testutil/dockerd/daemon.go +++ b/util/testutil/dockerd/daemon.go @@ -2,6 +2,7 @@ package dockerd import ( "bytes" + "fmt" "io" "os" "os/exec" @@ -137,6 +138,7 @@ func (d *Daemon) StartWithError(daemonLogs map[string]*bytes.Buffer, providedArg d.cmd.Stderr = &lockingWriter{Writer: b} } + fmt.Fprintf(d.cmd.Stderr, "> startCmd %v %+v\n", time.Now(), d.cmd.String()) if err := d.cmd.Start(); err != nil { return errors.Wrapf(err, "[%s] could not start daemon container", d.id) } diff --git a/util/testutil/integration/dockerd.go b/util/testutil/integration/dockerd.go index c4083cf9bd21..a0168b45e797 100644 --- a/util/testutil/integration/dockerd.go +++ b/util/testutil/integration/dockerd.go @@ -1,7 +1,6 @@ package integration import ( - "bytes" "context" "encoding/json" "io" @@ -141,9 +140,8 @@ func (c moby) New(ctx context.Context, cfg *BackendConfig) (b Backend, cl func() } deferF.append(d.StopWithError) - logs := map[string]*bytes.Buffer{} if err := waitUnix(d.Sock(), 5*time.Second); err != nil { - return nil, nil, errors.Errorf("dockerd did not start up: %q, %s", err, formatLogs(logs)) + return nil, nil, errors.Errorf("dockerd did not start up: %q, %s", err, formatLogs(cfg.Logs)) } dockerAPI, err := client.NewClientWithOpts(client.WithHost(d.Sock()))