From 99c6887ddfe63eeeef5c77628c7e4da8035b3ccd Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 2 Oct 2025 23:06:07 -0700 Subject: [PATCH 1/2] remotecache: skip result on not-found error Signed-off-by: Tonis Tiigi --- cache/remotecache/azblob/importer.go | 5 ++-- cache/remotecache/gha/gha.go | 5 ++-- cache/remotecache/v1/chains.go | 3 +- solver/exporter.go | 41 ++++++++++++++++------------ 4 files changed, 31 insertions(+), 23 deletions(-) diff --git a/cache/remotecache/azblob/importer.go b/cache/remotecache/azblob/importer.go index 87ad50b525c3..a98d4ca136aa 100644 --- a/cache/remotecache/azblob/importer.go +++ b/cache/remotecache/azblob/importer.go @@ -11,6 +11,7 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/pkg/labels" + cerrdefs "github.com/containerd/errdefs" "github.com/moby/buildkit/cache/remotecache" v1 "github.com/moby/buildkit/cache/remotecache/v1" "github.com/moby/buildkit/session" @@ -214,7 +215,7 @@ type ciProvider struct { func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) { if dgst != p.desc.Digest { - return content.Info{}, errors.Errorf("content not found %s", dgst) + return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst) } if p.checked { @@ -234,7 +235,7 @@ func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info } if !exists { - return content.Info{}, errors.Errorf("blob %s not found", dgst) + return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst) } p.checked = true diff --git a/cache/remotecache/gha/gha.go b/cache/remotecache/gha/gha.go index c7ef08be73dc..5e2e0191443a 100644 --- a/cache/remotecache/gha/gha.go +++ b/cache/remotecache/gha/gha.go @@ -14,6 +14,7 @@ import ( "github.com/containerd/containerd/v2/core/content" "github.com/containerd/containerd/v2/pkg/labels" + cerrdefs "github.com/containerd/errdefs" "github.com/moby/buildkit/cache/remotecache" v1 "github.com/moby/buildkit/cache/remotecache/v1" "github.com/moby/buildkit/session" @@ -440,7 +441,7 @@ type ciProvider struct { func (p *ciProvider) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) { if dgst != p.desc.Digest { - return content.Info{}, errors.Errorf("content not found %s", dgst) + return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst) } if _, err := p.loadEntry(ctx, p.desc); err != nil { @@ -465,7 +466,7 @@ func (p *ciProvider) loadEntry(ctx context.Context, desc ocispecs.Descriptor) (* return nil, err } if ce == nil { - return nil, errors.Errorf("blob %s not found", desc.Digest) + return nil, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", desc.Digest) } if p.entries == nil { p.entries = make(map[digest.Digest]*actionscache.Entry) diff --git a/cache/remotecache/v1/chains.go b/cache/remotecache/v1/chains.go index 7f80399e53d2..1a89400b5cc3 100644 --- a/cache/remotecache/v1/chains.go +++ b/cache/remotecache/v1/chains.go @@ -10,6 +10,7 @@ import ( "github.com/cespare/xxhash/v2" "github.com/containerd/containerd/v2/core/content" + cerrdefs "github.com/containerd/errdefs" "github.com/moby/buildkit/session" "github.com/moby/buildkit/solver" digest "github.com/opencontainers/go-digest" @@ -246,7 +247,7 @@ func (p DescriptorProviderPair) Info(ctx context.Context, dgst digest.Digest) (c return p.InfoProvider.Info(ctx, dgst) } if dgst != p.Descriptor.Digest { - return content.Info{}, errors.Errorf("content not found %s", dgst) + return content.Info{}, errors.Wrapf(cerrdefs.ErrNotFound, "blob %s", dgst) } return content.Info{ Digest: p.Descriptor.Digest, diff --git a/solver/exporter.go b/solver/exporter.go index 3f6dc1ccf5a4..e49ba9712f08 100644 --- a/solver/exporter.go +++ b/solver/exporter.go @@ -5,6 +5,7 @@ import ( "errors" "slices" + cerrdefs "github.com/containerd/errdefs" digest "github.com/opencontainers/go-digest" ) @@ -189,24 +190,28 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach if (remote == nil || opt.CompressionOpt != nil) && opt.Mode != CacheExportModeRemoteOnly { res, err := cm.results.Load(ctx, res) if err != nil { - return nil, err - } - remotes, err := opt.ResolveRemotes(ctx, res) - if err != nil { - return nil, err - } - res.Release(context.TODO()) - if remote == nil && len(remotes) > 0 { - remote, remotes = remotes[0], remotes[1:] // pop the first element - } - if opt.CompressionOpt != nil { - for _, r := range remotes { // record all remaining remotes as well - results = append(results, CacheExportResult{ - CreatedAt: v.CreatedAt, - Result: r, - EdgeVertex: k.vtx, - EdgeIndex: k.output, - }) + if !errors.Is(err, cerrdefs.ErrNotFound) { + return nil, err + } + remote = nil + } else { + remotes, err := opt.ResolveRemotes(ctx, res) + if err != nil { + return nil, err + } + res.Release(context.TODO()) + if remote == nil && len(remotes) > 0 { + remote, remotes = remotes[0], remotes[1:] // pop the first element + } + if opt.CompressionOpt != nil { + for _, r := range remotes { // record all remaining remotes as well + results = append(results, CacheExportResult{ + CreatedAt: v.CreatedAt, + Result: r, + EdgeVertex: k.vtx, + EdgeIndex: k.output, + }) + } } } } From 77aa49d063ea2ddbce32db3f6688643b5cefcb53 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Thu, 2 Oct 2025 23:18:39 -0700 Subject: [PATCH 2/2] solver: avoid failing cache export on subbranch error Don't fail whole cache export on subbranch error. This behavior changed in v0.25, but while before error was not returned, the cache chains were either too agressively dropped or the whole exported cache chain got corrupted. Signed-off-by: Tonis Tiigi --- solver/exporter.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/solver/exporter.go b/solver/exporter.go index e49ba9712f08..6ade2f44ce11 100644 --- a/solver/exporter.go +++ b/solver/exporter.go @@ -237,7 +237,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach for _, dep := range deps { rec, err := dep.CacheKey.Exporter.ExportTo(ctx, t, opt) if err != nil { - return nil, err + continue } for _, r := range rec { srcs[i] = append(srcs[i], CacheLink{Src: r, Selector: string(dep.Selector)}) @@ -249,7 +249,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach for _, de := range e.edge.secondaryExporters { recs, err := de.cacheKey.CacheKey.Exporter.ExportTo(mainCtx, t, opt) if err != nil { - return nil, nil + continue } for _, r := range recs { srcs[de.index] = append(srcs[de.index], CacheLink{Src: r, Selector: de.cacheKey.Selector.String()}) @@ -266,6 +266,14 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach } } + // validate deps are present + for _, deps := range srcs { + if len(deps) == 0 { + res[e] = nil + return res[e], nil + } + } + if v != nil && len(deps) == 0 { cm := v.cacheManager key := cm.getID(v.key)