Skip to content

feat: use buildkit API for clean up build cache within system prune #1360

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AkihiroSuda merged 1 commit into from
Dec 16, 2022
Merged

feat: use buildkit API for clean up build cache within system prune #1360

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 66 additions & 1 deletion cmd/nerdctl/system_prune.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@
package main

import (
"context"
"fmt"
"strings"

buildkitclient "github.com/moby/buildkit/client"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
Expand Down Expand Up @@ -72,6 +74,7 @@ func systemPruneAction(cmd *cobra.Command, args []string) error {
}
msg += `
- all images without at least one container associated to them
- all build cache
`
msg += "\nAre you sure you want to continue? [y/N] "
fmt.Fprintf(cmd.OutOrStdout(), "WARNING! %s", msg)
Expand Down Expand Up @@ -99,5 +102,67 @@ func systemPruneAction(cmd *cobra.Command, args []string) error {
return err
}
}
return imagePrune(ctx, cmd, client)
if err := imagePrune(ctx, cmd, client); err != nil {
return nil
}
prunedObjects, err := buildCachePrune(ctx, cmd, all)
Comment thread
STRRL marked this conversation as resolved.
if err != nil {
return err
}

if len(prunedObjects) > 0 {
fmt.Fprintln(cmd.OutOrStdout(), "Deleted build cache objects:")
for _, item := range prunedObjects {
fmt.Fprintln(cmd.OutOrStdout(), item.ID)
}
}

// TODO: print total reclaimed space

return nil
}

type cacheUsageInfo struct {
ID string
Size int64
}

func buildCachePrune(ctx context.Context, cmd *cobra.Command, pruneAll bool) ([]cacheUsageInfo, error) {
Comment thread
STRRL marked this conversation as resolved.
buildkitHost, err := getBuildkitHost(cmd)
if err != nil {
return nil, err
}
opts := []buildkitclient.ClientOpt{buildkitclient.WithFailFast()}
client, err := buildkitclient.New(ctx, buildkitHost, opts)
if err != nil {
return nil, err
}
var pruneOpts []buildkitclient.PruneOption
if pruneAll {
pruneOpts = append(pruneOpts, buildkitclient.PruneAll)
}

var usageInfos []buildkitclient.UsageInfo
usageCh := make(chan buildkitclient.UsageInfo)

go func() {
for item := range usageCh {
usageInfos = append(usageInfos, item)
}
}()

err = client.Prune(ctx, usageCh, pruneOpts...)
close(usageCh)
if err != nil {
return nil, err
}

result := make([]cacheUsageInfo, len(usageInfos))
for i, item := range usageInfos {
Comment thread
STRRL marked this conversation as resolved.
result[i] = cacheUsageInfo{
ID: item.ID,
Size: item.Size,
}
}
return result, nil
}
39 changes: 39 additions & 0 deletions cmd/nerdctl/system_prune_linux_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,21 @@
package main

import (
"bytes"
"fmt"
"io"
"os"
"os/exec"
"strings"
"testing"

"github.com/containerd/nerdctl/pkg/buildkitutil"
"github.com/containerd/nerdctl/pkg/testutil"
"github.com/sirupsen/logrus"
)

func TestSystemPrune(t *testing.T) {
testutil.RequiresBuild(t)
base := testutil.NewBase(t)
base.Cmd("container", "prune", "-f").AssertOK()
base.Cmd("network", "prune", "-f").AssertOK()
Expand Down Expand Up @@ -51,4 +59,35 @@ func TestSystemPrune(t *testing.T) {
base.Cmd("ps", "-a").AssertNoOut(tID)
base.Cmd("network", "ls").AssertNoOut(nID)
base.Cmd("images").AssertNoOut("alpine")

if testutil.GetTarget() != testutil.Nerdctl {
t.Skip("test skipped for buildkitd is not available with docker-compatible tests")
}

buildctlBinary, err := buildkitutil.BuildctlBinary()
if err != nil {
t.Fatal(err)
}
host, err := buildkitutil.GetBuildkitHost(testutil.Namespace)
if err != nil {
t.Fatal(err)
}

buildctlArgs := buildkitutil.BuildctlBaseArgs(host)
buildctlArgs = append(buildctlArgs, "du")
logrus.Debugf("running %s %v", buildctlBinary, buildctlArgs)
buildctlCmd := exec.Command(buildctlBinary, buildctlArgs...)
buildctlCmd.Env = os.Environ()
stdout := bytes.NewBuffer(nil)
buildctlCmd.Stdout = stdout
if err := buildctlCmd.Run(); err != nil {
t.Fatal(err)
}
readAll, err := io.ReadAll(stdout)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(readAll), "Total:\t\t0B") {
t.Errorf("buildkit cache is not pruned: %s", string(readAll))
}
}
29 changes: 24 additions & 5 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,18 @@ require (
github.com/docker/go-units v0.5.0
github.com/fahedouch/go-logrotate v0.1.2
github.com/fatih/color v1.13.0
github.com/fluent/fluent-logger-golang v1.9.0
github.com/hashicorp/go-multierror v1.1.1
github.com/ipfs/go-cid v0.3.2
github.com/ipfs/go-ipfs-files v0.2.0
github.com/ipfs/go-ipfs-http-client v0.4.0
github.com/ipfs/interface-go-ipfs-core v0.8.0
github.com/klauspost/compress v1.15.12
github.com/mattn/go-isatty v0.0.16
github.com/mitchellh/mapstructure v1.5.0
github.com/moby/buildkit v0.10.6
github.com/moby/sys/mount v0.3.3
github.com/moby/sys/signal v0.7.0
github.com/multiformats/go-multiaddr v0.8.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b
Expand All @@ -54,6 +59,7 @@ require (
golang.org/x/sync v0.1.0
golang.org/x/sys v0.3.0
golang.org/x/term v0.3.0
golang.org/x/text v0.5.0
gopkg.in/yaml.v3 v3.0.1
gotest.tools/v3 v3.4.0
)
Expand All @@ -71,15 +77,22 @@ require (
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
github.com/distribution/distribution/v3 v3.0.0-20221103125252-ebfa2a0ac0a9 // indirect
github.com/djherbis/times v1.5.0 // indirect
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/docker-credential-helpers v0.6.4 // indirect
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
github.com/fluent/fluent-logger-golang v1.9.0
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/gofrs/flock v0.8.1 // indirect
github.com/gogo/googleapis v1.4.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
// golang-lru contains patented ARC cache code, but we do not compile it in. Verified using hack/verify-no-patent.sh .
github.com/hashicorp/golang-lru v0.5.4 // indirect
Expand Down Expand Up @@ -107,7 +120,6 @@ require (
github.com/ipld/go-codec-dagpb v1.3.2 // indirect
github.com/ipld/go-ipld-prime v0.16.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
github.com/klauspost/compress v1.15.12
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
// NOTE: P2P image distribution (IPFS) is completely optional. Your host is NOT connected to any P2P network, unless you opt in to install and run IPFS daemon.
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
Expand All @@ -120,11 +132,9 @@ require (
github.com/miekg/pkcs11 v1.1.1 // indirect
github.com/minio/sha256-simd v1.0.0 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.5.0
github.com/moby/locker v1.0.1 // indirect
github.com/moby/sys/mountinfo v0.6.2 // indirect
github.com/moby/sys/sequential v0.0.0-20220829095930-b22ba8a69b30 // indirect
github.com/moby/sys/signal v0.7.0
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.1.0 // indirect
Expand All @@ -146,6 +156,7 @@ require (
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect
github.com/tinylib/msgp v1.1.6 // indirect
github.com/tonistiigi/fsutil v0.0.0-20220315205639-9ed612626da3 // indirect
github.com/urfave/cli v1.22.9 // indirect
github.com/vbatts/tar-split v0.11.2 // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158 // indirect
Expand All @@ -154,11 +165,16 @@ require (
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 // indirect
go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.32.0 // indirect
go.opentelemetry.io/otel v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0 // indirect
go.opentelemetry.io/otel/sdk v1.7.0 // indirect
go.opentelemetry.io/otel/trace v1.7.0 // indirect
go.opentelemetry.io/proto/otlp v0.16.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.23.0 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/text v0.5.0
golang.org/x/tools v0.1.12 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect
Expand All @@ -168,3 +184,6 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
lukechampine.com/blake3 v1.1.7 // indirect
)

// we use this specific version of docker because of the buildkit depends on it: https://github.com/moby/buildkit/blob/c9a0f4d2de095591e742d7f411d9ed36a03a1c4e/go.mod#L153
replace github.com/docker/docker => github.com/docker/docker v20.10.3-0.20220831131523-b5a0d7a188ac+incompatible
Loading