From 705a9bcfcaa72b124972c0b395a76e79e8207aa1 Mon Sep 17 00:00:00 2001
From: Tonis Tiigi <tonistiigi@gmail.com>
Date: Wed, 28 May 2025 20:59:25 -0700
Subject: [PATCH] solver: handle not found error on cache export

When a cleanup happens while a build is running it
may cause boltdb record to be deleted while that record
would have been used for cache export of the running build,
causing the cache export to fail with Not Found error. This
is possible because when cache is loaded to snapshots then
snapshots are reference counted, protecting against them being
released or deleted, but when cache key is just intermediary,
the query result just gets saved and is used later.

It should be safe to ignore this error and skip over the result
if it doesn't exist anymore.

Note that cache export also gets called on provenance
creation because cache chains are used for finding remote
descriptors for build steps. This was the more likely
codepath for hitting this error.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>
---
 solver/exporter.go      | 51 ++++++++++++++++++++++++++++++++---------
 solver/exporter_test.go | 34 +++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 11 deletions(-)
 create mode 100644 solver/exporter_test.go

diff --git a/solver/exporter.go b/solver/exporter.go
index 5f040f75d03b..b1f758f92fb4 100644
--- a/solver/exporter.go
+++ b/solver/exporter.go
@@ -2,6 +2,8 @@ package solver
 
 import (
 	"context"
+	"errors"
+	"slices"
 
 	digest "github.com/opencontainers/go-digest"
 )
@@ -102,18 +104,29 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
 		exportRecord = true
 	}
 
-	if e.record == nil && exportRecord {
-		e.record = getBestResult(e.records)
-	}
+	records := slices.Clone(e.records)
+	slices.SortStableFunc(records, compareCacheRecord)
 
 	var remote *Remote
-	if v := e.record; v != nil && exportRecord && addRecord {
+	var i int
+	for exportRecord && addRecord {
 		var variants []CacheExporterRecord
-
+		v := e.record
+		if v == nil {
+			if i < len(records) {
+				v = records[i]
+				i++
+			} else {
+				break
+			}
+		}
 		cm := v.cacheManager
 		key := cm.getID(v.key)
 		res, err := cm.backend.Load(key, v.ID)
 		if err != nil {
+			if errors.Is(err, ErrNotFound) {
+				continue
+			}
 			return nil, err
 		}
 
@@ -160,6 +173,7 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
 			}
 		}
 		allRec = append(allRec, variants...)
+		break
 	}
 
 	if remote != nil && opt.Mode == CacheExportModeMin {
@@ -228,13 +242,28 @@ func (e *exporter) ExportTo(ctx context.Context, t CacheExporterTarget, opt Cach
 }
 
 func getBestResult(records []*CacheRecord) *CacheRecord {
-	var rec *CacheRecord
-	for _, r := range records {
-		if rec == nil || rec.CreatedAt.Before(r.CreatedAt) || (rec.CreatedAt.Equal(r.CreatedAt) && rec.Priority < r.Priority) {
-			rec = r
-		}
+	records = slices.Clone(records)
+	slices.SortStableFunc(records, compareCacheRecord)
+	if len(records) == 0 {
+		return nil
+	}
+	return records[0]
+}
+
+func compareCacheRecord(a, b *CacheRecord) int {
+	if a == nil && b == nil {
+		return 0
+	}
+	if a == nil {
+		return 1
+	}
+	if b == nil {
+		return -1
+	}
+	if v := b.CreatedAt.Compare(a.CreatedAt); v != 0 {
+		return v
 	}
-	return rec
+	return a.Priority - b.Priority
 }
 
 type mergedExporter struct {
diff --git a/solver/exporter_test.go b/solver/exporter_test.go
new file mode 100644
index 000000000000..3534e41fa4a0
--- /dev/null
+++ b/solver/exporter_test.go
@@ -0,0 +1,34 @@
+package solver
+
+import (
+	"slices"
+	"testing"
+	"time"
+)
+
+func TestCompareCacheRecord(t *testing.T) {
+	now := time.Now()
+	a := &CacheRecord{CreatedAt: now, Priority: 1}
+	b := &CacheRecord{CreatedAt: now, Priority: 2}
+	c := &CacheRecord{CreatedAt: now.Add(1 * time.Second), Priority: 1}
+	d := &CacheRecord{CreatedAt: now.Add(-1 * time.Second), Priority: 1}
+
+	records := []*CacheRecord{b, nil, d, a, c, nil}
+	slices.SortFunc(records, compareCacheRecord)
+
+	names := map[*CacheRecord]string{
+		a:   "a",
+		b:   "b",
+		c:   "c",
+		d:   "d",
+		nil: "nil",
+	}
+	var got []string
+	for _, r := range records {
+		got = append(got, names[r])
+	}
+	want := []string{"c", "a", "b", "d", "nil", "nil"}
+	if !slices.Equal(got, want) {
+		t.Fatalf("unexpected order: got %v, want %v", got, want)
+	}
+}