From d9c44ac0f54fffcc9ebcde3846e2f7c54e848910 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Tue, 31 Jan 2023 10:16:29 +0000 Subject: [PATCH 01/19] progress: keep color enabled with NO_COLOR empty The NO_COLOR specification says that color should be disabled if NO_COLOR is set *and not empty*. Signed-off-by: Justin Chadwell (cherry picked from commit 03903488248f9818a6863e836affe18bf584bd65) --- util/progress/progressui/init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/progress/progressui/init.go b/util/progress/progressui/init.go index f21072214825..75f0cb83d1de 100644 --- a/util/progress/progressui/init.go +++ b/util/progress/progressui/init.go @@ -14,7 +14,7 @@ var colorError aec.ANSI func init() { // As recommended on https://no-color.org/ - if _, ok := os.LookupEnv("NO_COLOR"); ok { + if v := os.Getenv("NO_COLOR"); v != "" { // nil values will result in no ANSI color codes being emitted. return } else if runtime.GOOS == "windows" { From 6b91118be759944fcf61925274dfe4e1ee22bdb5 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Tue, 31 Jan 2023 11:29:33 +0000 Subject: [PATCH 02/19] sbom: create tmp directory for scanner image Signed-off-by: Justin Chadwell (cherry picked from commit 25c190d9e913180d41e865da4c9ffcfaeb3dda6a) --- frontend/attestations/sbom/sbom.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/attestations/sbom/sbom.go b/frontend/attestations/sbom/sbom.go index b4446aed45f0..113797b2139c 100644 --- a/frontend/attestations/sbom/sbom.go +++ b/frontend/attestations/sbom/sbom.go @@ -78,6 +78,8 @@ func CreateSBOMScanner(ctx context.Context, resolver llb.ImageMetaResolver, scan } runscan := llb.Image(scanner).Run(runOpts...) + runscan.AddMount("/tmp", llb.Scratch(), llb.Tmpfs()) + runscan.AddMount(path.Join(srcDir, "core", CoreSBOMName), ref, llb.Readonly) for k, extra := range extras { runscan.AddMount(path.Join(srcDir, "extras", ExtraSBOMPrefix+k), extra, llb.Readonly) From 04e63cbbf0fe4f825b5eebe432f6e9abb54d5bd8 Mon Sep 17 00:00:00 2001 From: Akihiro Suda Date: Tue, 31 Jan 2023 23:17:36 +0900 Subject: [PATCH 03/19] SOURCE_DATE_EPOCH: drop timezone The timezone information is not exposed to the uint64 representation, but exposed to RFC3339 representation. Signed-off-by: Akihiro Suda (cherry picked from commit 5849ab118ba888a9e367f583d29655188197577e) --- exporter/util/epoch/parse.go | 2 +- frontend/dockerfile/builder/build.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exporter/util/epoch/parse.go b/exporter/util/epoch/parse.go index 9d581ed9136c..63f806e1b76a 100644 --- a/exporter/util/epoch/parse.go +++ b/exporter/util/epoch/parse.go @@ -60,6 +60,6 @@ func parseTime(key, value string) (*time.Time, error) { if err != nil { return nil, errors.Wrapf(err, "invalid %s: %s", key, err) } - tm := time.Unix(sde, 0) + tm := time.Unix(sde, 0).UTC() return &tm, nil } diff --git a/frontend/dockerfile/builder/build.go b/frontend/dockerfile/builder/build.go index f9dd3643cd63..aafd9c9a73f3 100644 --- a/frontend/dockerfile/builder/build.go +++ b/frontend/dockerfile/builder/build.go @@ -1133,6 +1133,6 @@ func parseSourceDateEpoch(v string) (*time.Time, error) { if err != nil { return nil, errors.Wrapf(err, "invalid SOURCE_DATE_EPOCH: %s", v) } - tm := time.Unix(sde, 0) + tm := time.Unix(sde, 0).UTC() return &tm, nil } From 4fe7d93b5b565e18890c17f9196e74d3473336a8 Mon Sep 17 00:00:00 2001 From: coryb Date: Tue, 31 Jan 2023 15:31:51 -0800 Subject: [PATCH 04/19] do not mount secrets that are optional and missing from solve opts Signed-off-by: coryb (cherry picked from commit b5afdec47c8de6de9325d74751c2f15a988ac450) --- client/client_test.go | 18 ++++++++++++++++-- solver/llbsolver/mounts/mount.go | 6 +++--- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index c1f87dc7d4b9..1aa3b87f2cbc 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -1137,9 +1137,9 @@ func testSecretMounts(t *testing.T, sb integration.Sandbox) { }, nil) require.NoError(t, err) - // test optional + // test optional, mount should not exist when secret not present in SolveOpt st = llb.Image("busybox:latest"). - Run(llb.Shlex(`echo secret2`), llb.AddSecret("/run/secrets/mysecret2", llb.SecretOptional)) + Run(llb.Shlex(`test ! -f /run/secrets/mysecret2`), llb.AddSecret("/run/secrets/mysecret2", llb.SecretOptional)) def, err = st.Marshal(sb.Context()) require.NoError(t, err) @@ -1176,6 +1176,20 @@ func testSecretMounts(t *testing.T, sb integration.Sandbox) { })}, }, nil) require.NoError(t, err) + + // test empty cert still creates secret file + st = llb.Image("busybox:latest"). + Run(llb.Shlex(`test -f /run/secrets/mysecret5`), llb.AddSecret("/run/secrets/mysecret5", llb.SecretID("mysecret"))) + + def, err = st.Marshal(sb.Context()) + require.NoError(t, err) + + _, err = c.Solve(sb.Context(), def, SolveOpt{ + Session: []session.Attachable{secretsprovider.FromMap(map[string][]byte{ + "mysecret": []byte(""), + })}, + }, nil) + require.NoError(t, err) } func testSecretEnv(t *testing.T, sb integration.Sandbox) { diff --git a/solver/llbsolver/mounts/mount.go b/solver/llbsolver/mounts/mount.go index 37bc8a602d71..2cfeaae7a213 100644 --- a/solver/llbsolver/mounts/mount.go +++ b/solver/llbsolver/mounts/mount.go @@ -251,14 +251,14 @@ func (mm *MountManager) getSecretMountable(ctx context.Context, m *pb.Mount, g s err = mm.sm.Any(ctx, g, func(ctx context.Context, _ string, caller session.Caller) error { dt, err = secrets.GetSecret(ctx, caller, id) if err != nil { - if errors.Is(err, secrets.ErrNotFound) && m.SecretOpt.Optional { - return nil - } return err } return nil }) if err != nil { + if errors.Is(err, secrets.ErrNotFound) && m.SecretOpt.Optional { + return nil, nil + } return nil, err } return &secretMount{mount: m, data: dt, idmap: mm.cm.IdentityMapping()}, nil From 522d28e82873b0454eed60ae0e8386b828b428c2 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 31 Jan 2023 22:54:20 -0800 Subject: [PATCH 05/19] =?UTF-8?q?cache:=20don=E2=80=99t=20link=20blobonly?= =?UTF-8?q?=20based=20on=20chainid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tonis Tiigi (cherry picked from commit 99bd0d80468747efbb62f38d4f9cb04c95c54927) --- cache/manager.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cache/manager.go b/cache/manager.go index 983f7cd5295f..d579a6007ba0 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -222,10 +222,8 @@ func (cm *cacheManager) GetByBlob(ctx context.Context, desc ocispecs.Descriptor, id := identity.NewID() snapshotID := chainID.String() - blobOnly := true if link != nil { snapshotID = link.getSnapshotID() - blobOnly = link.getBlobOnly() go link.Release(context.TODO()) } @@ -289,7 +287,7 @@ func (cm *cacheManager) GetByBlob(ctx context.Context, desc ocispecs.Descriptor, rec.queueChainID(chainID) rec.queueBlobChainID(blobChainID) rec.queueSnapshotID(snapshotID) - rec.queueBlobOnly(blobOnly) + rec.queueBlobOnly(true) rec.queueMediaType(desc.MediaType) rec.queueBlobSize(desc.Size) rec.appendURLs(desc.URLs) From e164caece25bab90e5e81417352f5b64b789c6c1 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 3 Feb 2023 18:39:24 +0100 Subject: [PATCH 06/19] docs: update syntax for labs channel in examples The dockerfile:1-labs frontend is now updated to 1.5-labs, which includes this feature Signed-off-by: Sebastiaan van Stijn (cherry picked from commit faf476047e0d13d93f4fe57068cdea1f451b1a01) --- frontend/dockerfile/docs/reference.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frontend/dockerfile/docs/reference.md b/frontend/dockerfile/docs/reference.md index e6d2809f30cc..0065a59eb062 100644 --- a/frontend/dockerfile/docs/reference.md +++ b/frontend/dockerfile/docs/reference.md @@ -1323,7 +1323,7 @@ ADD [--keep-git-dir=] ``` ```dockerfile -# syntax=docker/dockerfile-upstream:master-labs +# syntax=docker/dockerfile:1-labs FROM alpine ADD --keep-git-dir=true https://github.com/moby/buildkit.git#v0.10.1 /buildkit ``` @@ -1331,9 +1331,11 @@ ADD --keep-git-dir=true https://github.com/moby/buildkit.git#v0.10.1 /buildkit The `--keep-git-dir=true` flag adds the `.git` directory. This flag defaults to false. ### Adding a private git repository + To add a private repo via SSH, create a Dockerfile with the following form: + ```dockerfile -# syntax = docker/dockerfile-upstream:master-labs +# syntax=docker/dockerfile:1-labs FROM alpine ADD git@git.example.com:foo/bar.git /bar ``` From 24c991bf9b755c13930b16aa7198a5ef60b0f8f4 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sat, 4 Feb 2023 03:29:20 +0100 Subject: [PATCH 07/19] frontend: fix typo in release script Signed-off-by: CrazyMax (cherry picked from commit 63e915e241db55f494f6ab60a78e0c821c8b8dcc) --- frontend/dockerfile/cmd/dockerfile-frontend/hack/release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release index 4de0c9c6b27f..053a986df680 100755 --- a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release +++ b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release @@ -23,7 +23,7 @@ parseTag() { fi local suffix=$(echo $1 | awk -F- '{print $NF}') local tagf=./frontend/dockerfile/release/$suffix/tags - if [ "$sufffix" == "$1" ] || [ ! -f $tagf ]; then + if [ "$suffix" == "$1" ] || [ ! -f $tagf ]; then suffix="mainline" fi From a980d83925cc3d48646667ac2a372bdcb6df9893 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sat, 4 Feb 2023 10:19:10 +0100 Subject: [PATCH 08/19] hack: treat unset variables as an error Signed-off-by: CrazyMax (cherry picked from commit 886b1dad2bbd4a9eabb24484ab64df9a1bee267a) --- frontend/dockerfile/cmd/dockerfile-frontend/hack/release | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release index 053a986df680..da0d91e14834 100755 --- a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release +++ b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release @@ -1,10 +1,10 @@ #!/usr/bin/env bash . $(dirname $0)/../../../../../hack/util -set -e +set -eu -: ${PLATFORMS=} -: ${DAILY_TARGETS=} +: "${PLATFORMS=}" +: "${DAILY_TARGETS=}" usage() { echo "$0 (master|tag|daily) (tag|channel) [push]" From bfe0729406b9e387db2c7dd2412231b3d1656740 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Tue, 7 Feb 2023 13:09:06 +0000 Subject: [PATCH 09/19] exporter: canonicalize sbom file paths during search Signed-off-by: Justin Chadwell (cherry picked from commit e4c7c6dc417d5079803d77dfa5b42fa626227e2f) --- exporter/containerimage/attestations.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/exporter/containerimage/attestations.go b/exporter/containerimage/attestations.go index 782c18733035..af3962393187 100644 --- a/exporter/containerimage/attestations.go +++ b/exporter/containerimage/attestations.go @@ -172,6 +172,8 @@ func newFileLayerFinder(target cache.ImmutableRef, remote *solver.Remote) (fileL // // find is not concurrency-safe. func (c *fileLayerFinder) find(ctx context.Context, s session.Group, filename string) (cache.ImmutableRef, *ocispecs.Descriptor, error) { + filename = filepath.Join("/", filename) + // return immediately if we've already found the layer containing filename if cache, ok := c.cache[filename]; ok { return cache.ref, &cache.desc, nil @@ -188,6 +190,8 @@ func (c *fileLayerFinder) find(ctx context.Context, s session.Group, filename st found := false for _, f := range files { + f = filepath.Join("/", f) + if strings.HasPrefix(f, ".wh.") { // skip whiteout files, we only care about file creations continue From 21200785c24fc31b8311a21cdd05bb63a2ce6c32 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Tue, 7 Feb 2023 13:09:37 +0000 Subject: [PATCH 10/19] exporter: fix file layer finder whiteout detection Signed-off-by: Justin Chadwell (cherry picked from commit 32dc0ece694692f0845737029f778b1ff25e2325) --- exporter/containerimage/attestations.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exporter/containerimage/attestations.go b/exporter/containerimage/attestations.go index af3962393187..d41a657bd7f7 100644 --- a/exporter/containerimage/attestations.go +++ b/exporter/containerimage/attestations.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "io/fs" + "path/filepath" "strings" intoto "github.com/in-toto/in-toto-golang/in_toto" @@ -192,7 +193,7 @@ func (c *fileLayerFinder) find(ctx context.Context, s session.Group, filename st for _, f := range files { f = filepath.Join("/", f) - if strings.HasPrefix(f, ".wh.") { + if strings.HasPrefix(filepath.Base(f), ".wh.") { // skip whiteout files, we only care about file creations continue } From 0ddf8016648252cff20332bed86e95824b7667b2 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Tue, 7 Feb 2023 15:50:44 +0000 Subject: [PATCH 11/19] exporter: fix supplement sboms on empty scratch layer Signed-off-by: Justin Chadwell (cherry picked from commit f3db1142cd3b4524926db5c1c41fd98de2691d0e) --- exporter/containerimage/attestations.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/exporter/containerimage/attestations.go b/exporter/containerimage/attestations.go index d41a657bd7f7..8d28675e07d8 100644 --- a/exporter/containerimage/attestations.go +++ b/exporter/containerimage/attestations.go @@ -31,6 +31,9 @@ var intotoPlatform ocispecs.Platform = ocispecs.Platform{ // supplementSBOM modifies SPDX attestations to include the file layers func supplementSBOM(ctx context.Context, s session.Group, target cache.ImmutableRef, targetRemote *solver.Remote, att exporter.Attestation) (exporter.Attestation, error) { + if target == nil { + return att, nil + } if att.Kind != gatewaypb.AttestationKindInToto { return att, nil } From f0c74da97c4764bb0979d4d5d40f47ed36a9eb74 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Tue, 7 Feb 2023 17:00:18 +0000 Subject: [PATCH 12/19] exporter: fix sbom supplement core detection Signed-off-by: Justin Chadwell (cherry picked from commit 257fb65c9cf0e1f63d98e326713b80b5224f10b5) --- exporter/containerimage/attestations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/containerimage/attestations.go b/exporter/containerimage/attestations.go index 8d28675e07d8..a41c6039f0ba 100644 --- a/exporter/containerimage/attestations.go +++ b/exporter/containerimage/attestations.go @@ -44,7 +44,7 @@ func supplementSBOM(ctx context.Context, s session.Group, target cache.Immutable if !ok { return att, nil } - if n, _, _ := strings.Cut(att.Path, "."); n != string(name) { + if n, _, _ := strings.Cut(filepath.Base(att.Path), "."); n != string(name) { return att, nil } From 95aaabda0d712938c349eedc54a36f153523ff8a Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Tue, 7 Feb 2023 17:02:21 +0000 Subject: [PATCH 13/19] client: add tests for layerID in comment field Signed-off-by: Justin Chadwell (cherry picked from commit 1c55dc2c58130810579dcc8f82a29d8ee5e4c792) --- client/client_test.go | 150 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/client/client_test.go b/client/client_test.go index 1aa3b87f2cbc..0fca38e23a70 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -62,6 +62,7 @@ import ( digest "github.com/opencontainers/go-digest" ocispecs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + spdx "github.com/spdx/tools-golang/spdx/v2_3" "github.com/stretchr/testify/require" "golang.org/x/crypto/ssh/agent" "golang.org/x/sync/errgroup" @@ -189,6 +190,7 @@ func TestIntegration(t *testing.T) { testAttestationBundle, testSBOMScan, testSBOMScanSingleRef, + testSBOMSupplements, testMultipleCacheExports, testMountStubsDirectory, testMountStubsTimestamp, @@ -8199,6 +8201,154 @@ EOF require.Subset(t, attest.Predicate, map[string]interface{}{"name": "fallback"}) } +func testSBOMSupplements(t *testing.T, sb integration.Sandbox) { + integration.CheckFeatureCompat(t, sb, integration.FeatureDirectPush, integration.FeatureSBOM) + requiresLinux(t) + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + + p := platforms.MustParse("linux/amd64") + pk := platforms.Format(p) + + frontend := func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + res := gateway.NewResult() + + // build image + st := llb.Scratch().File( + llb.Mkfile("/foo", 0600, []byte{}), + ) + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + r, err := c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + ref, err := r.SingleRef() + if err != nil { + return nil, err + } + _, err = ref.ToState() + if err != nil { + return nil, err + } + res.AddRef(pk, ref) + + expPlatforms := &exptypes.Platforms{ + Platforms: []exptypes.Platform{{ID: pk, Platform: p}}, + } + dt, err := json.Marshal(expPlatforms) + if err != nil { + return nil, err + } + res.AddMeta(exptypes.ExporterPlatformsKey, dt) + + // build attestations + doc := spdx.Document{ + SPDXIdentifier: "DOCUMENT", + Files: []*spdx.File{ + { + // foo exists... + FileSPDXIdentifier: "SPDXRef-File-foo", + FileName: "/foo", + }, + { + // ...but bar doesn't + FileSPDXIdentifier: "SPDXRef-File-bar", + FileName: "/bar", + }, + }, + } + docBytes, err := json.Marshal(doc) + if err != nil { + return nil, err + } + st = llb.Scratch(). + File(llb.Mkfile("/result.spdx", 0600, docBytes)) + def, err = st.Marshal(ctx) + if err != nil { + return nil, err + } + r, err = c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + refAttest, err := r.SingleRef() + if err != nil { + return nil, err + } + _, err = ref.ToState() + if err != nil { + return nil, err + } + + res.AddAttestation(pk, gateway.Attestation{ + Kind: gatewaypb.AttestationKindInToto, + Ref: refAttest, + Path: "/result.spdx", + InToto: result.InTotoAttestation{ + PredicateType: intoto.PredicateSPDX, + }, + Metadata: map[string][]byte{ + result.AttestationSBOMCore: []byte("result"), + }, + }) + + return res, nil + } + + // test the default fallback scanner + target := registry + "/buildkit/testsbom:latest" + _, err = c.Build(sb.Context(), SolveOpt{ + FrontendAttrs: map[string]string{ + "attest:sbom": "", + }, + Exports: []ExportEntry{ + { + Type: ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }, + }, + }, "", frontend, nil) + require.NoError(t, err) + + desc, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + + imgs, err := testutil.ReadImages(sb.Context(), provider, desc) + require.NoError(t, err) + require.Equal(t, 2, len(imgs.Images)) + + att := imgs.Find("unknown/unknown") + attest := struct { + intoto.StatementHeader + Predicate spdx.Document + }{} + require.NoError(t, json.Unmarshal(att.LayersRaw[0], &attest)) + require.Equal(t, "https://in-toto.io/Statement/v0.1", attest.Type) + require.Equal(t, intoto.PredicateSPDX, attest.PredicateType) + + require.Equal(t, "DOCUMENT", string(attest.Predicate.SPDXIdentifier)) + require.Len(t, attest.Predicate.Files, 2) + require.Equal(t, attest.Predicate.Files[0].FileName, "/foo") + require.Regexp(t, "^layerID: sha256:", attest.Predicate.Files[0].FileComment) + require.Equal(t, attest.Predicate.Files[1].FileName, "/bar") + require.Empty(t, attest.Predicate.Files[1].FileComment) +} + func testMultipleCacheExports(t *testing.T, sb integration.Sandbox) { integration.CheckFeatureCompat(t, sb, integration.FeatureMultiCacheExport) c, err := New(sb.Context(), sb.Address()) From b924f0b5ce5e97d43bd3400a92c1ae77ccf1ab5c Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Tue, 7 Feb 2023 17:48:56 -0800 Subject: [PATCH 14/19] cache: add fallback for snapshotID In older BuildKit versions snapshotID was not always set if record was not created with GetByBlob method. Old code defaulted to cache record ID in that case but that broke with the metadata interface refactor. Signed-off-by: Tonis Tiigi (cherry picked from commit 2c0c429530e20aac317a65c72acec42f0da44b0f) --- cache/metadata.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cache/metadata.go b/cache/metadata.go index d6410fd554db..82209a93c0f9 100644 --- a/cache/metadata.go +++ b/cache/metadata.go @@ -251,7 +251,13 @@ func (md *cacheMetadata) queueMediaType(str string) error { } func (md *cacheMetadata) getSnapshotID() string { - return md.GetString(keySnapshot) + sid := md.GetString(keySnapshot) + // Note that historic buildkit releases did not always set the snapshot ID. + // Fallback to record ID is needed for old build cache compatibility. + if sid == "" { + return md.ID() + } + return sid } func (md *cacheMetadata) queueSnapshotID(str string) error { From b31ccd0417a05ca119b62de9791bb71060c07a5d Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 10 Feb 2023 19:23:39 -0800 Subject: [PATCH 15/19] gha: avoid range requests with too big offset Signed-off-by: Tonis Tiigi (cherry picked from commit 488791a5a1498caf7cdf1a2b3390398635cf1025) --- cache/remotecache/gha/gha.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cache/remotecache/gha/gha.go b/cache/remotecache/gha/gha.go index 45cacb5b8332..f36693d3b08d 100644 --- a/cache/remotecache/gha/gha.go +++ b/cache/remotecache/gha/gha.go @@ -5,6 +5,7 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "sync" "time" @@ -371,6 +372,13 @@ type readerAt struct { desc ocispecs.Descriptor } +func (r *readerAt) ReadAt(p []byte, off int64) (int, error) { + if off >= r.desc.Size { + return 0, io.EOF + } + return r.ReaderAtCloser.ReadAt(p, off) +} + func (r *readerAt) Size() int64 { return r.desc.Size } From b66611781dec5ae926b0a6f5ab2a742fbf0f8331 Mon Sep 17 00:00:00 2001 From: Justin Chadwell Date: Thu, 19 Jan 2023 10:52:30 +0000 Subject: [PATCH 16/19] dockerfile: release frontend for i386 platform Signed-off-by: Justin Chadwell --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8d97ec4ef7f..9ec900822d5f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -498,7 +498,7 @@ jobs: run: | ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" "${{ needs.frontend-base.outputs.tag }}" "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" env: - PLATFORMS: ${{ env.PLATFORMS }},linux/mips,linux/mipsle,linux/mips64,linux/mips64le + PLATFORMS: ${{ env.PLATFORMS }},linux/386,linux/mips,linux/mipsle,linux/mips64,linux/mips64le CACHE_FROM: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }} CACHE_TO: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }} - @@ -507,5 +507,5 @@ jobs: run: | ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" labs "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" env: - PLATFORMS: ${{ env.PLATFORMS }},linux/mips,linux/mipsle,linux/mips64,linux/mips64le + PLATFORMS: ${{ env.PLATFORMS }},linux/386,linux/mips,linux/mipsle,linux/mips64,linux/mips64le CACHE_FROM: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }} From 2d0740f048271fd53787da9649dd8b47c7475125 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 2 Feb 2023 16:04:31 +0100 Subject: [PATCH 17/19] ci: create matrix for building frontend image Signed-off-by: CrazyMax --- .github/workflows/build.yml | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9ec900822d5f..902c3e7958b5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -444,8 +444,8 @@ jobs: if: github.event_name != 'schedule' outputs: typ: ${{ steps.prep.outputs.typ }} - tag: ${{ steps.prep.outputs.tag }} push: ${{ steps.prep.outputs.push }} + matrix: ${{ steps.prep.outputs.matrix }} steps: - name: Prepare @@ -462,14 +462,30 @@ jobs: PUSH=push fi echo "typ=${TYP}" >>${GITHUB_OUTPUT} - echo "tag=${TAG}" >>${GITHUB_OUTPUT} echo "push=${PUSH}" >>${GITHUB_OUTPUT} + if [ "${TYP}" = "master" ]; then + echo "matrix=$(jq -cn --arg tag "$TAG" '[$tag, "labs"]')" >>${GITHUB_OUTPUT} + else + echo "matrix=$(jq -cn --arg tag "$TAG" '[$tag]')" >>${GITHUB_OUTPUT} + fi frontend-image: runs-on: ubuntu-20.04 if: github.event_name != 'schedule' needs: [frontend-base, test] + strategy: + fail-fast: false + matrix: + tag: ${{ fromJson(needs.frontend-base.outputs.matrix) }} steps: + - + name: Prepare + run: | + if [ "${{ matrix.tag }}" = "labs" ]; then + echo "CACHE_SCOPE=frontend-labs" >>${GITHUB_ENV} + else + echo "CACHE_SCOPE=frontend-mainline" >>${GITHUB_ENV} + fi - name: Checkout uses: actions/checkout@v3 @@ -494,18 +510,10 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build ${{ needs.frontend-base.outputs.typ }}/${{ needs.frontend-base.outputs.tag }} - run: | - ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" "${{ needs.frontend-base.outputs.tag }}" "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" - env: - PLATFORMS: ${{ env.PLATFORMS }},linux/386,linux/mips,linux/mipsle,linux/mips64,linux/mips64le - CACHE_FROM: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }} - CACHE_TO: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }} - - - name: Build ${{ needs.frontend-base.outputs.typ }}/labs - if: needs.frontend-base.outputs.typ == 'master' + name: Build run: | - ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" labs "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" + ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" "${{ matrix.tag }}" "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" env: PLATFORMS: ${{ env.PLATFORMS }},linux/386,linux/mips,linux/mipsle,linux/mips64,linux/mips64le - CACHE_FROM: type=gha,scope=frontend-${{ needs.frontend-base.outputs.typ }} + CACHE_FROM: type=gha,scope=${{ env.CACHE_SCOPE }} + CACHE_TO: type=gha,scope=${{ env.CACHE_SCOPE }} From 821f130af2bd5ee0dc83a22e6e2eb377e51c076d Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 27 Jan 2023 21:45:11 +0100 Subject: [PATCH 18/19] hack: do not set attest flags when exporting to docker Signed-off-by: CrazyMax --- hack/images | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hack/images b/hack/images index 8c4564caa27e..6df942765254 100755 --- a/hack/images +++ b/hack/images @@ -39,12 +39,15 @@ if [[ "$TAG" == "local" ]]; then fi fi +attestFlags="$(buildAttestFlags)" + outputFlag="--output=type=image,push=false" if [ "$PUSH" = "push" ]; then outputFlag="--output=type=image,buildinfo-attrs=true,push=true" fi if [ -n "$localmode" ]; then outputFlag="--output=type=docker,buildinfo-attrs=true" + attestFlags="" fi targetFlag="" @@ -88,5 +91,5 @@ for tagName in $tagNames; do tagFlags="$tagFlags--tag=$tagName " done -buildxCmd build $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $(buildAttestFlags) \ +buildxCmd build $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $attestFlags \ $currentcontext From 66e05870676a51dc713d35eb84abed8cfc17c078 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 2 Feb 2023 15:40:05 +0100 Subject: [PATCH 19/19] hack: do not cache some stages on release Signed-off-by: CrazyMax --- .github/workflows/build.yml | 3 +++ .../cmd/dockerfile-frontend/hack/release | 14 ++++++++++---- hack/images | 8 +++++++- hack/release-tar | 8 +++++++- 4 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 902c3e7958b5..9fbcac794fed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -390,6 +390,7 @@ jobs: run: | ./hack/images "${{ needs.release-base.outputs.tag }}" "$REPO_SLUG_TARGET" "${{ needs.release-base.outputs.push }}" env: + RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }} TARGET: ${{ matrix.target-stage }} CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} type=gha,scope=image${{ matrix.target-stage }} CACHE_TO: type=gha,scope=image${{ matrix.target-stage }} @@ -419,6 +420,7 @@ jobs: run: | ./hack/release-tar "${{ needs.release-base.outputs.tag }}" release-out env: + RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }} PLATFORMS: ${{ env.PLATFORMS }},darwin/amd64,darwin/arm64,windows/amd64,windows/arm64 CACHE_FROM: type=gha,scope=${{ env.CACHE_GHA_SCOPE_BINARIES }} type=gha,scope=${{ env.CACHE_GHA_SCOPE_CROSS }} - @@ -514,6 +516,7 @@ jobs: run: | ./frontend/dockerfile/cmd/dockerfile-frontend/hack/release "${{ needs.frontend-base.outputs.typ }}" "${{ matrix.tag }}" "$DF_REPO_SLUG_TARGET" "${{ needs.frontend-base.outputs.push }}" env: + RELEASE: ${{ startsWith(github.ref, 'refs/tags/v') }} PLATFORMS: ${{ env.PLATFORMS }},linux/386,linux/mips,linux/mipsle,linux/mips64,linux/mips64le CACHE_FROM: type=gha,scope=${{ env.CACHE_SCOPE }} CACHE_TO: type=gha,scope=${{ env.CACHE_SCOPE }} diff --git a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release index da0d91e14834..8d71f83e8def 100755 --- a/frontend/dockerfile/cmd/dockerfile-frontend/hack/release +++ b/frontend/dockerfile/cmd/dockerfile-frontend/hack/release @@ -3,6 +3,7 @@ . $(dirname $0)/../../../../../hack/util set -eu +: "${RELEASE=false}" : "${PLATFORMS=}" : "${DAILY_TARGETS=}" @@ -70,6 +71,11 @@ if [ "$PUSH" = "push" ]; then pushFlag="push=true" fi +nocacheFilterFlag="" +if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then + nocacheFilterFlag="--no-cache-filter=base" +fi + case $TYP in "master") tagf=./frontend/dockerfile/release/$TAG/tags @@ -84,7 +90,7 @@ case $TYP in pushTag=${pushTag}-$TAG fi - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $(buildAttestFlags) \ + buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag $(buildAttestFlags) \ --build-arg "CHANNEL=$TAG" \ --build-arg "BUILDTAGS=$buildTags" \ --output "type=image,name=$REPO:$pushTag,buildinfo-attrs=true,$pushFlag" \ @@ -101,7 +107,7 @@ case $TYP in fi buildTags=$(cat $tagf) - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $(buildAttestFlags) \ + buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag $(buildAttestFlags) \ --build-arg "CHANNEL=$TAG" \ --build-arg "BUILDTAGS=$buildTags" \ --output "type=image,\"name=$publishedNames\",buildinfo-attrs=true,$pushFlag" \ @@ -127,7 +133,7 @@ case $TYP in tmp=$(mktemp -d -t buildid.XXXXXXXXXX) dt=$(date +%Y%m%d) - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags \ + buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag \ --target "buildid" \ --build-arg "CHANNEL=$TAG" \ --build-arg "BUILDTAGS=$buildTags" \ @@ -141,7 +147,7 @@ case $TYP in buildid=$(cat $tmp/buildid) echo "buildid: $buildid" - buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $(buildAttestFlags) \ + buildxCmd build $platformFlag $cacheFromFlags $cacheToFlags $nocacheFilterFlag $(buildAttestFlags) \ --build-arg "CHANNEL=$TAG" \ --build-arg "BUILDTAGS=$buildTags" \ --output "type=image,name=$REPO:$dt-$TAG,buildinfo-attrs=true,$pushFlag" \ diff --git a/hack/images b/hack/images index 6df942765254..cbaf0eb16f2c 100755 --- a/hack/images +++ b/hack/images @@ -7,6 +7,7 @@ PUSH=$3 . $(dirname $0)/util set -eu -o pipefail +: ${RELEASE=false} : ${PLATFORMS=} : ${TARGET=} @@ -91,5 +92,10 @@ for tagName in $tagNames; do tagFlags="$tagFlags--tag=$tagName " done -buildxCmd build $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $attestFlags \ +nocacheFilterFlag="" +if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then + nocacheFilterFlag="--no-cache-filter=git,buildkit-export,gobuild-base" +fi + +buildxCmd build $platformFlag $targetFlag $importCacheFlags $exportCacheFlags $tagFlags $outputFlag $nocacheFilterFlag $attestFlags \ $currentcontext diff --git a/hack/release-tar b/hack/release-tar index 549fe5f3f2c6..308243f72096 100755 --- a/hack/release-tar +++ b/hack/release-tar @@ -6,6 +6,7 @@ OUT=$2 . $(dirname $0)/util set -eu -o pipefail +: ${RELEASE=false} : ${PLATFORMS=} usage() { @@ -22,9 +23,14 @@ if [ -n "$PLATFORMS" ]; then platformFlag="--platform=$PLATFORMS" fi +nocacheFilterFlag="" +if [[ "$RELEASE" = "true" ]] && [[ "$GITHUB_ACTIONS" = "true" ]]; then + nocacheFilterFlag="--no-cache-filter=git,gobuild-base" +fi + output=$(mktemp -d -t buildkit-output.XXXXXXXXXX) -buildxCmd build $platformFlag $cacheFromFlags $(buildAttestFlags) \ +buildxCmd build $platformFlag $cacheFromFlags $nocacheFilterFlag $(buildAttestFlags) \ --build-arg "BUILDKIT_MULTI_PLATFORM=true" \ --target release \ --output "type=local,dest=$output" \