From 9f888e05c123f971abebb14a278001bf11a9af0f Mon Sep 17 00:00:00 2001 From: Sertac Ozercan Date: Wed, 11 Mar 2026 21:45:16 +0000 Subject: [PATCH] feat: add OCI artifact export via postprocessor Add support for exporting OCI artifacts through a postprocessor model. Frontends signal artifact metadata on the build result, and an artifact processor runs between the build and export phases to assemble an OCI layout via LLB. The transformed result is then handled by any built-in exporter (image, oci, local). The artifact frontend runs in a standard frontend sandbox, provenance is captured for all postprocessor steps, and layer paths are normalized and validated to prevent path traversal. Signed-off-by: Sertac Ozercan --- client/client_test.go | 406 ++++++++++++++++++ cmd/buildkitd/main.go | 2 + control/control.go | 2 + exporter/containerimage/annotations.go | 21 + exporter/containerimage/artifact.go | 87 ++++ exporter/containerimage/export.go | 182 ++++++++ exporter/containerimage/exptypes/artifact.go | 11 + exporter/containerimage/exptypes/types.go | 14 + exporter/containerimage/writer.go | 266 ++++++++++++ exporter/oci/export.go | 132 ++++++ frontend/artifact/artifact.go | 239 +++++++++++ frontend/artifact/artifact_test.go | 63 +++ solver/llbsolver/proc/artifact.go | 149 +++++++ solver/llbsolver/proc/artifact_test.go | 69 +++ solver/llbsolver/provenance/capture.go | 115 +++++ solver/llbsolver/provenance/predicate.go | 14 + solver/llbsolver/provenance/types/types.go | 6 +- .../llbsolver/provenance/types/types_test.go | 14 + solver/llbsolver/solver.go | 26 ++ util/push/push.go | 2 +- 20 files changed, 1818 insertions(+), 2 deletions(-) create mode 100644 exporter/containerimage/artifact.go create mode 100644 exporter/containerimage/exptypes/artifact.go create mode 100644 frontend/artifact/artifact.go create mode 100644 frontend/artifact/artifact_test.go create mode 100644 solver/llbsolver/proc/artifact.go create mode 100644 solver/llbsolver/proc/artifact_test.go diff --git a/client/client_test.go b/client/client_test.go index 33344b601d62..b658896f2b75 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -208,6 +208,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){ testPullWithLayerLimit, testExportAnnotations, testExportAnnotationsMediaTypes, + testOCIArtifactPostprocessor, testExportAttestationsOCIArtifact, testExportAttestationsImageManifest, testExportedImageLabels, @@ -10014,6 +10015,411 @@ func testExportAnnotationsMediaTypes(t *testing.T, sb integration.Sandbox) { require.Equal(t, ocispecs.MediaTypeImageIndex, imgs2.Index.MediaType) } +func testOCIArtifactPostprocessor(t *testing.T, sb integration.Sandbox) { + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + const ( + artifactType = "application/vnd.buildkit.test.artifact.v1" + configMediaType = "application/vnd.buildkit.test.config.v1+json" + layerMediaType = "application/vnd.buildkit.test.layer.v1" + artifactFile = "artifact.txt" + ) + + artifactPayload := []byte("artifact payload") + expectedConfigDigest := digest.FromBytes([]byte("{}")) + + artifactFrontend := func(configType, layerType string) gateway.BuildFunc { + return func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + st := integration.UnixOrWindows(llb.Scratch(), llb.Image("nanoserver")).File( + llb.Mkfile(artifactFile, 0600, artifactPayload), + ) + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + r, err := c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + ref, err := r.SingleRef() + if err != nil { + return nil, err + } + + layers, err := json.Marshal([]exptypes.ArtifactLayer{{ + Path: artifactFile, + MediaType: layerType, + Annotations: map[string]string{ + ocispecs.AnnotationTitle: artifactFile, + }, + }}) + if err != nil { + return nil, err + } + + res := gateway.NewResult() + res.SetRef(ref) + res.AddMeta(exptypes.ExporterArtifactKey, []byte("true")) + res.AddMeta(exptypes.ExporterArtifactTypeKey, []byte(artifactType)) + res.AddMeta(exptypes.ExporterArtifactConfigTypeKey, []byte(configType)) + res.AddMeta(exptypes.ExporterArtifactLayersKey, layers) + return res, nil + } + } + + decodeDescriptor := func(t *testing.T, encoded string) ocispecs.Descriptor { + t.Helper() + + dt, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + + var desc ocispecs.Descriptor + require.NoError(t, json.Unmarshal(dt, &desc)) + return desc + } + + type provenanceStatement struct { + intoto.StatementHeader + Predicate provenancetypes.ProvenancePredicateSLSA1 `json:"predicate"` + } + + findArtifactDescriptors := func(t *testing.T, manifests []ocispecs.Descriptor) (ocispecs.Descriptor, ocispecs.Descriptor) { + t.Helper() + + var artifactDesc, attestationDesc ocispecs.Descriptor + for _, desc := range manifests { + switch { + case desc.Platform == nil: + artifactDesc = desc + case platforms.Format(*desc.Platform) == "unknown/unknown": + attestationDesc = desc + } + } + + require.NotEmpty(t, artifactDesc.Digest) + require.NotEmpty(t, attestationDesc.Digest) + return artifactDesc, attestationDesc + } + + t.Run("local", func(t *testing.T) { + dir := t.TempDir() + _, err := c.Build(sb.Context(), SolveOpt{ + Exports: []ExportEntry{{ + Type: ExporterLocal, + OutputDir: dir, + }}, + }, "", artifactFrontend(configMediaType, layerMediaType), nil) + require.NoError(t, err) + + _, err = os.Stat(filepath.Join(dir, artifactFile)) + require.ErrorIs(t, err, os.ErrNotExist) + + layoutData, err := os.ReadFile(filepath.Join(dir, ocispecs.ImageLayoutFile)) + require.NoError(t, err) + var layout ocispecs.ImageLayout + require.NoError(t, json.Unmarshal(layoutData, &layout)) + require.Equal(t, ocispecs.ImageLayoutVersion, layout.Version) + + indexData, err := os.ReadFile(filepath.Join(dir, ocispecs.ImageIndexFile)) + require.NoError(t, err) + var index ocispecs.Index + require.NoError(t, json.Unmarshal(indexData, &index)) + require.Len(t, index.Manifests, 1) + require.Equal(t, artifactType, index.Manifests[0].ArtifactType) + require.Equal(t, ocispecs.MediaTypeImageManifest, index.Manifests[0].MediaType) + + manifestData, err := os.ReadFile(filepath.Join(dir, ocispecs.ImageBlobsDir, "sha256", index.Manifests[0].Digest.Encoded())) + require.NoError(t, err) + var manifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(manifestData, &manifest)) + require.Equal(t, artifactType, manifest.ArtifactType) + require.Equal(t, configMediaType, manifest.Config.MediaType) + require.Equal(t, expectedConfigDigest, manifest.Config.Digest) + require.Len(t, manifest.Layers, 1) + require.Equal(t, layerMediaType, manifest.Layers[0].MediaType) + require.Equal(t, artifactFile, manifest.Layers[0].Annotations[ocispecs.AnnotationTitle]) + + layerData, err := os.ReadFile(filepath.Join(dir, ocispecs.ImageBlobsDir, "sha256", manifest.Layers[0].Digest.Encoded())) + require.NoError(t, err) + require.Equal(t, artifactPayload, layerData) + }) + + t.Run("local-provenance", func(t *testing.T) { + dir := t.TempDir() + _, err := c.Build(sb.Context(), SolveOpt{ + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=max,version=v1", + }, + Exports: []ExportEntry{{ + Type: ExporterLocal, + OutputDir: dir, + }}, + }, "", artifactFrontend(configMediaType, layerMediaType), nil) + require.NoError(t, err) + + provDt, err := os.ReadFile(filepath.Join(dir, "provenance.json")) + require.NoError(t, err) + + var stmt provenanceStatement + require.NoError(t, json.Unmarshal(provDt, &stmt)) + require.Len(t, stmt.Predicate.BuildDefinition.InternalParameters.Postprocess, 1) + require.Equal(t, "artifact.v0", stmt.Predicate.BuildDefinition.InternalParameters.Postprocess[0].Frontend) + require.Equal(t, artifactType, stmt.Predicate.BuildDefinition.InternalParameters.Postprocess[0].Args[exptypes.ExporterArtifactTypeKey]) + }) + + t.Run("oci", func(t *testing.T) { + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter) + + dir := t.TempDir() + out := filepath.Join(dir, "artifact.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + resp, err := c.Build(sb.Context(), SolveOpt{ + Exports: []ExportEntry{{ + Type: ExporterOCI, + Output: fixedWriteCloser(outW), + }}, + }, "", artifactFrontend(configMediaType, layerMediaType), nil) + require.NoError(t, err) + + desc := decodeDescriptor(t, resp.ExporterResponse[exptypes.ExporterImageDescriptorKey]) + require.Equal(t, artifactType, desc.ArtifactType) + require.Equal(t, expectedConfigDigest.String(), resp.ExporterResponse[exptypes.ExporterImageConfigDigestKey]) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + + m, err := testutil.ReadTarToMap(dt, false) + require.NoError(t, err) + + var index ocispecs.Index + require.NoError(t, json.Unmarshal(m[ocispecs.ImageIndexFile].Data, &index)) + require.Len(t, index.Manifests, 1) + require.Equal(t, artifactType, index.Manifests[0].ArtifactType) + + var manifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+index.Manifests[0].Digest.Hex()].Data, &manifest)) + require.Equal(t, artifactType, manifest.ArtifactType) + require.Equal(t, configMediaType, manifest.Config.MediaType) + require.Equal(t, expectedConfigDigest, manifest.Config.Digest) + }) + + t.Run("oci-inline-provenance", func(t *testing.T) { + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter) + + dir := t.TempDir() + out := filepath.Join(dir, "artifact-with-provenance.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + resp, err := c.Build(sb.Context(), SolveOpt{ + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=max,inline-only=true,version=v1", + }, + Exports: []ExportEntry{{ + Type: ExporterOCI, + Output: fixedWriteCloser(outW), + Attrs: map[string]string{ + "annotation-index.ai": "artifact index", + "annotation-index-descriptor.aid": "artifact index descriptor", + "annotation-manifest.am": "artifact manifest", + "annotation-manifest-descriptor.amd": "artifact manifest descriptor", + }, + }}, + }, "", artifactFrontend(configMediaType, layerMediaType), nil) + require.NoError(t, err) + + desc := decodeDescriptor(t, resp.ExporterResponse[exptypes.ExporterImageDescriptorKey]) + require.Equal(t, ocispecs.MediaTypeImageIndex, desc.MediaType) + require.Equal(t, expectedConfigDigest.String(), resp.ExporterResponse[exptypes.ExporterImageConfigDigestKey]) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + + m, err := testutil.ReadTarToMap(dt, false) + require.NoError(t, err) + + var layout ocispecs.Index + require.NoError(t, json.Unmarshal(m[ocispecs.ImageIndexFile].Data, &layout)) + require.Len(t, layout.Manifests, 1) + require.Equal(t, "artifact index descriptor", layout.Manifests[0].Annotations["aid"]) + + var index ocispecs.Index + require.NoError(t, json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+layout.Manifests[0].Digest.Hex()].Data, &index)) + require.Equal(t, "artifact index", index.Annotations["ai"]) + require.Len(t, index.Manifests, 2) + + artifactDesc, attestationDesc := findArtifactDescriptors(t, index.Manifests) + require.Equal(t, artifactType, artifactDesc.ArtifactType) + require.Equal(t, "artifact manifest descriptor", artifactDesc.Annotations["amd"]) + require.Equal(t, "unknown/unknown", platforms.Format(*attestationDesc.Platform)) + + var artifactManifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+artifactDesc.Digest.Hex()].Data, &artifactManifest)) + require.Equal(t, artifactType, artifactManifest.ArtifactType) + require.Equal(t, "artifact manifest", artifactManifest.Annotations["am"]) + + var attestationManifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+attestationDesc.Digest.Hex()].Data, &attestationManifest)) + require.Equal(t, "application/vnd.docker.attestation.manifest.v1+json", attestationManifest.ArtifactType) + require.NotNil(t, attestationManifest.Subject) + require.Equal(t, artifactDesc.Digest, attestationManifest.Subject.Digest) + require.Len(t, attestationManifest.Layers, 1) + + var stmt provenanceStatement + require.NoError(t, json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+attestationManifest.Layers[0].Digest.Hex()].Data, &stmt)) + require.Len(t, stmt.Predicate.BuildDefinition.InternalParameters.Postprocess, 1) + require.Equal(t, "artifact.v0", stmt.Predicate.BuildDefinition.InternalParameters.Postprocess[0].Frontend) + }) + + t.Run("image", func(t *testing.T) { + workers.CheckFeatureCompat(t, sb, workers.FeatureDirectPush) + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + target := registry + "/buildkit/testartifactpostprocessor:latest" + resp, err := c.Build(sb.Context(), SolveOpt{ + Exports: []ExportEntry{{ + Type: ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }}, + }, "", artifactFrontend(configMediaType, layerMediaType), nil) + require.NoError(t, err) + + desc := decodeDescriptor(t, resp.ExporterResponse[exptypes.ExporterImageDescriptorKey]) + require.Equal(t, artifactType, desc.ArtifactType) + require.Equal(t, desc.Digest.String(), resp.ExporterResponse[exptypes.ExporterImageDigestKey]) + require.Equal(t, expectedConfigDigest.String(), resp.ExporterResponse[exptypes.ExporterImageConfigDigestKey]) + + _, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + manifestData, err := content.ReadBlob(sb.Context(), provider, desc) + require.NoError(t, err) + + var manifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(manifestData, &manifest)) + require.Equal(t, artifactType, manifest.ArtifactType) + require.Equal(t, configMediaType, manifest.Config.MediaType) + require.Equal(t, expectedConfigDigest, manifest.Config.Digest) + require.Len(t, manifest.Layers, 1) + require.Equal(t, layerMediaType, manifest.Layers[0].MediaType) + }) + + t.Run("image-provenance", func(t *testing.T) { + workers.CheckFeatureCompat(t, sb, workers.FeatureDirectPush) + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + target := registry + "/buildkit/testartifactpostprocessor-provenance:latest" + resp, err := c.Build(sb.Context(), SolveOpt{ + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=max,version=v1", + }, + Exports: []ExportEntry{{ + Type: ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }}, + }, "", artifactFrontend(configMediaType, layerMediaType), nil) + require.NoError(t, err) + + desc := decodeDescriptor(t, resp.ExporterResponse[exptypes.ExporterImageDescriptorKey]) + require.Equal(t, ocispecs.MediaTypeImageIndex, desc.MediaType) + require.Equal(t, desc.Digest.String(), resp.ExporterResponse[exptypes.ExporterImageDigestKey]) + require.Equal(t, expectedConfigDigest.String(), resp.ExporterResponse[exptypes.ExporterImageConfigDigestKey]) + + _, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + + indexData, err := content.ReadBlob(sb.Context(), provider, desc) + require.NoError(t, err) + + var index ocispecs.Index + require.NoError(t, json.Unmarshal(indexData, &index)) + require.Len(t, index.Manifests, 2) + + artifactDesc, attestationDesc := findArtifactDescriptors(t, index.Manifests) + require.Equal(t, artifactType, artifactDesc.ArtifactType) + require.Equal(t, "unknown/unknown", platforms.Format(*attestationDesc.Platform)) + require.Equal(t, attestation.DockerAnnotationReferenceTypeDefault, attestationDesc.Annotations[attestation.DockerAnnotationReferenceType]) + require.Equal(t, artifactDesc.Digest.String(), attestationDesc.Annotations[attestation.DockerAnnotationReferenceDigest]) + + artifactManifestData, err := content.ReadBlob(sb.Context(), provider, artifactDesc) + require.NoError(t, err) + var artifactManifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(artifactManifestData, &artifactManifest)) + require.Equal(t, artifactType, artifactManifest.ArtifactType) + require.Equal(t, configMediaType, artifactManifest.Config.MediaType) + require.Equal(t, expectedConfigDigest, artifactManifest.Config.Digest) + + attestationManifestData, err := content.ReadBlob(sb.Context(), provider, attestationDesc) + require.NoError(t, err) + var attestationManifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(attestationManifestData, &attestationManifest)) + require.Equal(t, "application/vnd.docker.attestation.manifest.v1+json", attestationManifest.ArtifactType) + require.NotNil(t, attestationManifest.Subject) + require.Equal(t, artifactDesc.Digest, attestationManifest.Subject.Digest) + require.Len(t, attestationManifest.Layers, 1) + + stmtData, err := content.ReadBlob(sb.Context(), provider, attestationManifest.Layers[0]) + require.NoError(t, err) + var stmt provenanceStatement + require.NoError(t, json.Unmarshal(stmtData, &stmt)) + require.Len(t, stmt.Predicate.BuildDefinition.InternalParameters.Postprocess, 1) + require.Equal(t, "artifact.v0", stmt.Predicate.BuildDefinition.InternalParameters.Postprocess[0].Frontend) + }) + + t.Run("validation", func(t *testing.T) { + for _, tc := range []struct { + name string + configType string + layerType string + errorContains string + }{ + { + name: "missing-config-media-type", + layerType: layerMediaType, + errorContains: "config descriptor mediaType is required", + }, + { + name: "missing-layer-media-type", + configType: configMediaType, + errorContains: "layer 0 descriptor mediaType is required", + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + _, err := c.Build(sb.Context(), SolveOpt{ + Exports: []ExportEntry{{ + Type: ExporterLocal, + OutputDir: dir, + }}, + }, "", artifactFrontend(tc.configType, tc.layerType), nil) + require.Error(t, err) + require.Contains(t, err.Error(), tc.errorContains) + }) + } + }) +} + func testExportAttestationsOCIArtifact(t *testing.T, sb integration.Sandbox) { testExportAttestations(t, sb, true) } diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index 7de17c050559..9ed0efd39ee8 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -34,6 +34,7 @@ import ( "github.com/moby/buildkit/control" "github.com/moby/buildkit/executor/oci" "github.com/moby/buildkit/frontend" + artifactfrontend "github.com/moby/buildkit/frontend/artifact" dockerfile "github.com/moby/buildkit/frontend/dockerfile/builder" "github.com/moby/buildkit/frontend/gateway" "github.com/moby/buildkit/frontend/gateway/forwarder" @@ -817,6 +818,7 @@ func newController(ctx context.Context, c *cli.Context, cfg *config.Config) (*co if cfg.Frontends.Dockerfile.Enabled == nil || *cfg.Frontends.Dockerfile.Enabled { frontends["dockerfile.v0"] = forwarder.NewGatewayForwarder(wc.Infos(), dockerfile.Build) } + frontends[artifactfrontend.Name] = forwarder.NewGatewayForwarder(wc.Infos(), artifactfrontend.Build) if cfg.Frontends.Gateway.Enabled == nil || *cfg.Frontends.Gateway.Enabled { gwfe, err := gateway.NewGatewayFrontend(wc.Infos(), cfg.Frontends.Gateway.AllowedRepositories) if err != nil { diff --git a/control/control.go b/control/control.go index e91f566c7aff..c62d2310f9a0 100644 --- a/control/control.go +++ b/control/control.go @@ -516,6 +516,8 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* procs = append(procs, proc.SBOMProcessor(ref.String(), useCache, resolveMode, params)) } + procs = append(procs, proc.ArtifactProcessor()) + if attrs, ok := attests["provenance"]; ok { var slsaVersion provenancetypes.ProvenanceSLSA params := make(map[string]string) diff --git a/exporter/containerimage/annotations.go b/exporter/containerimage/annotations.go index 1bf10f6c15db..a0e1e9906402 100644 --- a/exporter/containerimage/annotations.go +++ b/exporter/containerimage/annotations.go @@ -88,6 +88,27 @@ func (ag AnnotationsGroup) Platform(p *ocispecs.Platform) *Annotations { return res } +// ResolveArtifactAnnotations validates and resolves annotations for OCI +// artifact exports. Artifact exports always produce a single manifest +// descriptor, unless attestations are attached, in which case the top-level +// descriptor becomes an index that wraps the artifact manifest and attestation +// manifests. +func ResolveArtifactAnnotations(ag AnnotationsGroup, hasIndex bool) (*Annotations, error) { + resolved := ag.Platform(nil) + if !hasIndex && (len(resolved.Index) > 0 || len(resolved.IndexDescriptor) > 0) { + return nil, errors.Errorf("index annotations are not supported for OCI artifact exports without attestations") + } + for platform, annotations := range ag { + if platform == "" { + continue + } + if len(annotations.Manifest) > 0 || len(annotations.ManifestDescriptor) > 0 { + return nil, errors.Errorf("platform-specific annotations are not supported for OCI artifact exports") + } + } + return resolved, nil +} + func (ag AnnotationsGroup) Merge(other AnnotationsGroup) AnnotationsGroup { if other == nil { return ag diff --git a/exporter/containerimage/artifact.go b/exporter/containerimage/artifact.go new file mode 100644 index 000000000000..e2dcc125e64f --- /dev/null +++ b/exporter/containerimage/artifact.go @@ -0,0 +1,87 @@ +package containerimage + +import ( + "strconv" + "strings" + + intoto "github.com/in-toto/in-toto-golang/in_toto" + "github.com/moby/buildkit/exporter" + "github.com/moby/buildkit/exporter/attestation" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/solver/result" + "github.com/moby/buildkit/util/purl" + digest "github.com/opencontainers/go-digest" + "github.com/package-url/packageurl-go" + "github.com/pkg/errors" +) + +// ResolveArtifactAttestations selects the single attestation group that applies +// to an OCI artifact export. If forceInline is false, only inline-only +// attestations are retained, matching the regular image exporter behavior. +func ResolveArtifactAttestations(src *exporter.Source, forceInline bool) ([]exporter.Attestation, error) { + if len(src.Attestations) == 0 { + return nil, nil + } + + attestationGroups := make(map[string][]exporter.Attestation, len(src.Attestations)) + for k, atts := range src.Attestations { + if !forceInline { + atts = attestation.Filter(atts, nil, map[string][]byte{ + result.AttestationInlineOnlyKey: []byte(strconv.FormatBool(true)), + }) + } + if len(atts) > 0 { + attestationGroups[k] = atts + } + } + if len(attestationGroups) == 0 { + return nil, nil + } + if len(attestationGroups) == 1 { + for _, atts := range attestationGroups { + return atts, nil + } + } + + ps, err := exptypes.ParsePlatforms(src.Metadata) + if err != nil { + return nil, err + } + if len(ps.Platforms) != 1 { + return nil, errors.Errorf("OCI artifact exports require exactly one attestation group, got %d", len(attestationGroups)) + } + + atts, ok := attestationGroups[ps.Platforms[0].ID] + if !ok { + return nil, errors.Errorf("OCI artifact export missing attestation group for %s", ps.Platforms[0].ID) + } + if len(attestationGroups) != 1 { + return nil, errors.Errorf("OCI artifact exports require exactly one attestation group, got %d", len(attestationGroups)) + } + return atts, nil +} + +// DefaultArtifactSubjects creates default in-toto subjects for registry-pushed +// OCI artifacts. Artifact exports are platform-less, so no platform qualifier +// is added to the generated purls. +func DefaultArtifactSubjects(imageNames string, dgst digest.Digest) ([]intoto.Subject, error) { + if imageNames == "" { + return nil, nil + } + + var subjects []intoto.Subject + for name := range strings.SplitSeq(imageNames, ",") { + if name == "" { + continue + } + pl, err := purl.RefToPURL(packageurl.TypeDocker, name, nil) + if err != nil { + return nil, err + } + subjects = append(subjects, intoto.Subject{ + Name: pl, + Digest: result.ToDigestMap(dgst), + }) + } + return subjects, nil +} diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 918b55e7cf29..63dd52b6e643 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -233,6 +233,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source } opts.Annotations = opts.Annotations.Merge(as) + // If the result contains a pre-built OCI layout (from ArtifactProcessor), + // use the dedicated OCI layout export path instead of the normal image commit. + if _, ok := src.Metadata[exptypes.ExporterOCILayoutKey]; ok { + return e.exportOCILayout(ctx, src, buildInfo, opts.Annotations) + } + ctx, done, err := leaseutil.WithLease(ctx, e.opt.LeaseManager, leaseutil.MakeTemporary) if err != nil { return nil, nil, nil, err @@ -435,6 +441,182 @@ func (e *imageExporterInstance) pushImage(ctx context.Context, src *exporter.Sou return push.Push(ctx, e.opt.SessionManager, sessionID, mprovider, e.opt.ImageWriter.ContentStore(), dgst, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest, annotations) } +// exportOCILayout handles exporting a pre-built OCI layout that was assembled +// by the ArtifactProcessor. It ingests the layout into the content store, +// stores the image if configured, and optionally pushes it. +func (e *imageExporterInstance) exportOCILayout(ctx context.Context, src *exporter.Source, buildInfo exporter.ExportBuildInfo, annotations AnnotationsGroup) (_ map[string]string, _ exporter.FinalizeFunc, descref exporter.DescriptorReference, err error) { + if e.unpack { + return nil, nil, nil, errors.New("unpack is not supported for OCI artifact layouts") + } + if e.opts.RewriteTimestamp { + return nil, nil, nil, errors.New("rewrite-timestamp is not supported for OCI artifact layouts") + } + + // Resolve the single ref from the source + var ref cache.ImmutableRef + if src.Ref != nil { + ref = src.Ref + } else if len(src.Refs) == 1 { + for _, r := range src.Refs { + ref = r + } + } + if ref == nil { + return nil, nil, nil, errors.New("OCI layout export requires exactly one ref") + } + + // Acquire a lease to protect content from GC during export + ctx, done, err := leaseutil.WithLease(ctx, e.opt.LeaseManager, leaseutil.MakeTemporary) + if err != nil { + return nil, nil, nil, err + } + defer func() { + if descref == nil { + done(context.WithoutCancel(ctx)) + } + }() + + if n, ok := src.Metadata["image.name"]; e.opts.ImageName == "*" && ok { + e.opts.ImageName = string(n) + } + + attests, err := ResolveArtifactAttestations(src, e.opts.ForceInlineAttestations) + if err != nil { + return nil, nil, nil, err + } + + resolvedAnnotations, err := ResolveArtifactAnnotations(annotations, len(attests) > 0) + if err != nil { + return nil, nil, nil, err + } + + desc, err := e.opt.ImageWriter.CommitOCILayout(ctx, ref, buildInfo.SessionID, resolvedAnnotations) + if err != nil { + return nil, nil, nil, err + } + if len(attests) > 0 { + subjects, err := DefaultArtifactSubjects(e.opts.ImageName, desc.Digest) + if err != nil { + return nil, nil, nil, err + } + desc, err = e.opt.ImageWriter.CommitAttestationsForDescriptor(ctx, &e.opts, *desc, buildInfo.SessionID, attests, subjects, resolvedAnnotations) + if err != nil { + return nil, nil, nil, err + } + } + + resp := make(map[string]string) + + nameCanonical := e.nameCanonical + if e.danglingPrefix != "" && (!e.danglingEmptyOnly || e.opts.ImageName == "") { + danglingImageName := e.danglingPrefix + "@" + desc.Digest.String() + if e.opts.ImageName != "" { + e.opts.ImageName += "," + danglingImageName + } else { + e.opts.ImageName = danglingImageName + nameCanonical = false + } + } + + // Collect names for finalize callback to push + var namesToPush []string + + if e.opts.ImageName != "" { + targetNames := strings.SplitSeq(e.opts.ImageName, ",") + for targetName := range targetNames { + if e.opt.Images != nil && e.store { + tagDone := progress.OneOff(ctx, "naming to "+targetName) + + // imageClientCtx is used for propagating the epoch to e.opt.Images.Update() and e.opt.Images.Create(). + // + // Ideally, we should be able to propagate the epoch via images.Image.CreatedAt. + // However, due to a bug of containerd, we are temporarily stuck with this workaround. + // https://github.com/containerd/containerd/issues/8322 + imageClientCtx := ctx + if e.opts.Epoch != nil { + imageClientCtx = epoch.WithSourceDateEpoch(imageClientCtx, e.opts.Epoch) + } + img := images.Image{ + Target: *desc, + // CreatedAt in images.Images is ignored due to a bug of containerd. + // See the comment lines for imageClientCtx. + } + + sfx := []string{""} + if nameCanonical && !strings.ContainsRune(targetName, '@') { + sfx = append(sfx, "@"+desc.Digest.String()) + } + for _, sfx := range sfx { + img.Name = targetName + sfx + for { // handle possible race between Update and Create + if _, err := e.opt.Images.Update(imageClientCtx, img); err != nil { + if !errors.Is(err, cerrdefs.ErrNotFound) { + return nil, nil, nil, tagDone(err) + } + + if _, err := e.opt.Images.Create(imageClientCtx, img); err != nil { + if !errors.Is(err, cerrdefs.ErrAlreadyExists) { + return nil, nil, nil, tagDone(err) + } + continue + } + } + break + } + } + tagDone(nil) + } + if e.push { + namesToPush = append(namesToPush, targetName) + } + } + resp[exptypes.ExporterImageNameKey] = e.opts.ImageName + } + + resp[exptypes.ExporterImageDigestKey] = desc.Digest.String() + if v, ok := desc.Annotations[exptypes.ExporterConfigDigestKey]; ok { + resp[exptypes.ExporterImageConfigDigestKey] = v + delete(desc.Annotations, exptypes.ExporterConfigDigestKey) + } + + dtdesc, err := json.Marshal(desc) + if err != nil { + return nil, nil, nil, err + } + resp[exptypes.ExporterImageDescriptorKey] = base64.StdEncoding.EncodeToString(dtdesc) + + // Transfer lease ownership to descref + descref = NewDescriptorReference(*desc, done) + + if len(namesToPush) == 0 { + return resp, nil, descref, nil + } + + // Create finalize callback for pushing + finalize := func(ctx context.Context) error { + for _, targetName := range namesToPush { + if err := e.pushOCILayout(ctx, buildInfo.SessionID, targetName, desc.Digest); err != nil { + var statusErr remoteserrors.ErrUnexpectedStatus + if errors.As(err, &statusErr) { + err = errutil.WithDetails(err) + } + return errors.Wrapf(err, "failed to push OCI layout to %v", targetName) + } + } + return nil + } + + return resp, finalize, descref, nil +} + +// pushOCILayout pushes an OCI artifact layout to a registry. +// Unlike pushImage, it does not need to resolve remote providers from cache refs +// because all blobs have already been ingested into the content store by CommitOCILayout. +func (e *imageExporterInstance) pushOCILayout(ctx context.Context, sessionID string, targetName string, dgst digest.Digest) error { + cs := e.opt.ImageWriter.ContentStore() + return push.Push(ctx, e.opt.SessionManager, sessionID, cs, cs, dgst, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest, nil) +} + func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image, src *exporter.Source, s session.Group) (err0 error) { matcher := platforms.Only(platforms.Normalize(platforms.DefaultSpec())) diff --git a/exporter/containerimage/exptypes/artifact.go b/exporter/containerimage/exptypes/artifact.go new file mode 100644 index 000000000000..d415e11d2442 --- /dev/null +++ b/exporter/containerimage/exptypes/artifact.go @@ -0,0 +1,11 @@ +package exptypes + +// ArtifactLayer describes a single layer in an OCI artifact. +// The frontend produces an array of these as JSON metadata, which the +// ArtifactProcessor passes to the OCI layout assembler container. +type ArtifactLayer struct { + // Path is the root-relative file path inside the artifact input reference. + Path string `json:"path"` + MediaType string `json:"mediaType"` + Annotations map[string]string `json:"annotations,omitempty"` +} diff --git a/exporter/containerimage/exptypes/types.go b/exporter/containerimage/exptypes/types.go index d7cc98b9485e..8ffc4be740f5 100644 --- a/exporter/containerimage/exptypes/types.go +++ b/exporter/containerimage/exptypes/types.go @@ -16,6 +16,15 @@ const ( ExporterImageDescriptorKey = "containerimage.descriptor" ExporterImageBaseConfigKey = "containerimage.base.config" ExporterPlatformsKey = "refs.platforms" + + ExporterArtifactKey = "containerimage.artifact" + ExporterArtifactTypeKey = "containerimage.artifact.type" + ExporterArtifactConfigTypeKey = "containerimage.artifact.config.mediatype" + ExporterArtifactLayersKey = "containerimage.artifact.layers" + ExporterPostprocessorsKey = "containerimage.postprocessors" + ExporterOCILayoutKey = "containerimage.oci-layout" + + PostprocessInputKey = "input" ) // KnownRefMetadataKeys are the subset of exporter keys that can be suffixed by @@ -38,3 +47,8 @@ type InlineCacheEntry struct { Data []byte } type InlineCache func(ctx context.Context) (*result.Result[*InlineCacheEntry], error) + +type PostprocessRequest struct { + Frontend string `json:"frontend"` + FrontendOpt map[string]string `json:"frontendOpt,omitempty"` +} diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index f0cddeab91be..5dbec06d65a2 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -5,6 +5,8 @@ import ( "context" "encoding/json" "fmt" + "maps" + "os" "reflect" "strconv" "strings" @@ -14,6 +16,7 @@ import ( "github.com/containerd/containerd/v2/core/diff" "github.com/containerd/containerd/v2/core/images" "github.com/containerd/containerd/v2/pkg/labels" + "github.com/containerd/continuity/fs" "github.com/containerd/platforms" intoto "github.com/in-toto/in-toto-golang/in_toto" "github.com/moby/buildkit/cache" @@ -660,6 +663,87 @@ func (ic *ImageWriter) commitAttestationsManifest(ctx context.Context, opts *Ima }, nil } +// CommitAttestationsForDescriptor wraps an existing artifact/image manifest +// descriptor in an OCI index alongside its attestation manifest. +func (ic *ImageWriter) CommitAttestationsForDescriptor(ctx context.Context, opts *ImageCommitOpts, target ocispecs.Descriptor, sessionID string, attestations []exporter.Attestation, defaultSubjects []intoto.Subject, annotations *Annotations) (*ocispecs.Descriptor, error) { + if len(attestations) == 0 { + return &target, nil + } + if annotations == nil { + annotations = &Annotations{} + } + + opts.EnableOCITypes(ctx, "attestations") + + attestations, err := attestation.Unbundle(ctx, session.NewGroup(sessionID), attestations) + if err != nil { + return nil, err + } + + stmts, err := attestation.MakeInTotoStatements(ctx, session.NewGroup(sessionID), attestations, defaultSubjects) + if err != nil { + return nil, err + } + + attestationDesc, err := ic.commitAttestationsManifest(ctx, opts, target, stmts, true) + if err != nil { + return nil, err + } + attestationDesc.Platform = &intotoPlatform + + indexTarget := target + if len(target.Annotations) > 0 { + indexTarget.Annotations = maps.Clone(target.Annotations) + delete(indexTarget.Annotations, exptypes.ExporterConfigDigestKey) + if len(indexTarget.Annotations) == 0 { + indexTarget.Annotations = nil + } + } + + idx := ocispecs.Index{ + MediaType: ocispecs.MediaTypeImageIndex, + Annotations: maps.Clone(annotations.Index), + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Manifests: []ocispecs.Descriptor{ + indexTarget, + *attestationDesc, + }, + } + idxLabels := map[string]string{ + "containerd.io/gc.ref.content.0": target.Digest.String(), + "containerd.io/gc.ref.content.1": attestationDesc.Digest.String(), + } + + idxBytes, err := json.MarshalIndent(idx, "", " ") + if err != nil { + return nil, errors.Wrap(err, "failed to marshal artifact attestation index") + } + + idxDigest := digest.FromBytes(idxBytes) + idxDesc := ocispecs.Descriptor{ + Digest: idxDigest, + Size: int64(len(idxBytes)), + MediaType: idx.MediaType, + Annotations: maps.Clone(annotations.IndexDescriptor), + } + if v, ok := target.Annotations[exptypes.ExporterConfigDigestKey]; ok { + if idxDesc.Annotations == nil { + idxDesc.Annotations = map[string]string{} + } + idxDesc.Annotations[exptypes.ExporterConfigDigestKey] = v + } + + done := progress.OneOff(ctx, "exporting attestation index "+idxDigest.String()) + if err := content.WriteBlob(ctx, ic.opt.ContentStore, idxDigest.String(), bytes.NewReader(idxBytes), idxDesc, content.WithLabels(idxLabels)); err != nil { + return nil, done(errors.Wrapf(err, "error writing attestation index blob %s", idxDigest)) + } + done(nil) + + return &idxDesc, nil +} + func (ic *ImageWriter) ContentStore() content.Store { return ic.opt.ContentStore } @@ -950,3 +1034,185 @@ func getRefMetadata(ref cache.ImmutableRef, limit int) []refMetadata { } return metas } + +// CommitOCILayout reads an OCI image layout from the given ref's filesystem, +// ingests all blobs (config, layers, manifest) into the content store, and +// returns the manifest descriptor. If annotations is non-nil, manifest and +// manifest-descriptor annotations are applied before ingestion. +func (ic *ImageWriter) CommitOCILayout(ctx context.Context, ref cache.ImmutableRef, sessionID string, annotations *Annotations) (*ocispecs.Descriptor, error) { + if ref == nil { + return nil, errors.New("cannot commit OCI layout: ref is nil") + } + + // Mount the ref to get filesystem access + mount, err := ref.Mount(ctx, true, session.NewGroup(sessionID)) + if err != nil { + return nil, errors.Wrap(err, "failed to mount ref for OCI layout") + } + lm := snapshot.LocalMounter(mount) + rootDir, err := lm.Mount() + if err != nil { + return nil, errors.Wrap(err, "failed to locally mount ref for OCI layout") + } + defer lm.Unmount() + + // Helper to read a small file safely from the mounted root + readFileFromRoot := func(p string) ([]byte, error) { + fp, err := fs.RootPath(rootDir, p) + if err != nil { + return nil, errors.Wrapf(err, "failed to resolve path %s", p) + } + return os.ReadFile(fp) + } + + // Helper to open a file safely from the mounted root for streaming + openFileFromRoot := func(p string) (*os.File, error) { + fp, err := fs.RootPath(rootDir, p) + if err != nil { + return nil, errors.Wrapf(err, "failed to resolve path %s", p) + } + return os.Open(fp) + } + blobPath := func(d digest.Digest) string { + return ocispecs.ImageBlobsDir + "/" + d.Algorithm().String() + "/" + d.Encoded() + } + + cs := ic.opt.ContentStore + + // 1. Read and validate oci-layout file + layoutData, err := readFileFromRoot(ocispecs.ImageLayoutFile) + if err != nil { + return nil, errors.Wrap(err, "failed to read oci-layout file") + } + var layout ocispecs.ImageLayout + if err := json.Unmarshal(layoutData, &layout); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal oci-layout") + } + if layout.Version != ocispecs.ImageLayoutVersion { + return nil, errors.Errorf("unsupported OCI image layout version: %s", layout.Version) + } + + // 2. Read index.json + indexData, err := readFileFromRoot(ocispecs.ImageIndexFile) + if err != nil { + return nil, errors.Wrap(err, "failed to read index.json") + } + var idx ocispecs.Index + if err := json.Unmarshal(indexData, &idx); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal index.json") + } + + // 3. Validate: must have exactly 1 manifest + if len(idx.Manifests) != 1 { + return nil, errors.Errorf("OCI layout index.json must contain exactly 1 manifest, got %d", len(idx.Manifests)) + } + mfstDesc := idx.Manifests[0] + if mfstDesc.MediaType == "" { + return nil, errors.New("OCI layout manifest descriptor mediaType is required") + } + + // 4. Read the manifest blob + mfstData, err := readFileFromRoot(blobPath(mfstDesc.Digest)) + if err != nil { + return nil, errors.Wrapf(err, "failed to read manifest blob %s", mfstDesc.Digest) + } + var mfst ocispecs.Manifest + if err := json.Unmarshal(mfstData, &mfst); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal manifest") + } + if mfst.Config.MediaType == "" { + return nil, errors.New("OCI layout config descriptor mediaType is required") + } + for i, layer := range mfst.Layers { + if layer.MediaType == "" { + return nil, errors.Errorf("OCI layout layer %d descriptor mediaType is required", i) + } + } + switch { + case mfstDesc.ArtifactType == "": + mfstDesc.ArtifactType = mfst.ArtifactType + case mfst.ArtifactType != "" && mfstDesc.ArtifactType != mfst.ArtifactType: + return nil, errors.New("OCI layout manifest descriptor artifactType does not match manifest artifactType") + } + + // Apply user-requested manifest annotations. This changes the manifest + // content, so we must re-serialize and recompute the digest/size. + if annotations != nil && len(annotations.Manifest) > 0 { + if mfst.Annotations == nil { + mfst.Annotations = make(map[string]string) + } + for k, v := range annotations.Manifest { + mfst.Annotations[k] = v + } + mfstData, err = json.Marshal(mfst) + if err != nil { + return nil, errors.Wrap(err, "failed to re-marshal manifest with annotations") + } + mfstDesc.Digest = digest.FromBytes(mfstData) + mfstDesc.Size = int64(len(mfstData)) + } + + if annotations != nil && len(annotations.ManifestDescriptor) > 0 { + if mfstDesc.Annotations == nil { + mfstDesc.Annotations = make(map[string]string) + } + for k, v := range annotations.ManifestDescriptor { + mfstDesc.Annotations[k] = v + } + } + + // 5. Ingest config blob + configData, err := readFileFromRoot(blobPath(mfst.Config.Digest)) + if err != nil { + return nil, errors.Wrapf(err, "failed to read config blob %s", mfst.Config.Digest) + } + configDone := progress.OneOff(ctx, "ingesting OCI layout config "+mfst.Config.Digest.String()) + if err := content.WriteBlob(ctx, cs, mfst.Config.Digest.String(), bytes.NewReader(configData), mfst.Config); err != nil { + return nil, configDone(errors.Wrapf(err, "failed to ingest config blob %s", mfst.Config.Digest)) + } + configDone(nil) + + // 6. Ingest layer blobs — stream from mounted files to avoid buffering + // large artifacts (e.g. multi-GB model files) entirely in memory. + for i, layer := range mfst.Layers { + layerFile, err := openFileFromRoot(blobPath(layer.Digest)) + if err != nil { + return nil, errors.Wrapf(err, "failed to open layer blob %s", layer.Digest) + } + layerDone := progress.OneOff(ctx, fmt.Sprintf("ingesting OCI layout layer %d/%d %s", i+1, len(mfst.Layers), layer.Digest.String())) + err = content.WriteBlob(ctx, cs, layer.Digest.String(), layerFile, layer) + layerFile.Close() + if err != nil { + return nil, layerDone(errors.Wrapf(err, "failed to ingest layer blob %s", layer.Digest)) + } + layerDone(nil) + } + + // 7. Ingest manifest blob with GC reference labels + gcLabels := map[string]string{ + "containerd.io/gc.ref.content.config": mfst.Config.Digest.String(), + } + for i, l := range mfst.Layers { + gcLabels[fmt.Sprintf("containerd.io/gc.ref.content.l.%d", i)] = l.Digest.String() + } + + mfstBlobDesc := ocispecs.Descriptor{ + Digest: mfstDesc.Digest, + Size: int64(len(mfstData)), + MediaType: mfstDesc.MediaType, + } + mfstDone := progress.OneOff(ctx, "ingesting OCI layout manifest "+mfstDesc.Digest.String()) + if err := content.WriteBlob(ctx, cs, mfstDesc.Digest.String(), bytes.NewReader(mfstData), mfstBlobDesc, content.WithLabels(gcLabels)); err != nil { + return nil, mfstDone(errors.Wrapf(err, "failed to ingest manifest blob %s", mfstDesc.Digest)) + } + mfstDone(nil) + + if mfstDesc.Annotations == nil { + mfstDesc.Annotations = make(map[string]string) + } + mfstDesc.Annotations[exptypes.ExporterConfigDigestKey] = mfst.Config.Digest.String() + + bklog.G(ctx).Debugf("committed OCI layout: manifest=%s config=%s layers=%d", mfstDesc.Digest, mfst.Config.Digest, len(mfst.Layers)) + + return &mfstDesc, nil +} diff --git a/exporter/oci/export.go b/exporter/oci/export.go index 24b96a338a1f..c30fc020a197 100644 --- a/exporter/oci/export.go +++ b/exporter/oci/export.go @@ -149,6 +149,21 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source } opts.Annotations = opts.Annotations.Merge(as) + if _, ok := src.Metadata[exptypes.ExporterOCILayoutKey]; ok { + if e.opt.Variant == VariantDocker { + return nil, nil, nil, errors.New("docker exporter does not support OCI artifact layouts") + } + attests, err := containerimage.ResolveArtifactAttestations(src, e.opts.ForceInlineAttestations) + if err != nil { + return nil, nil, nil, err + } + resolvedAnnotations, err := containerimage.ResolveArtifactAnnotations(opts.Annotations, len(attests) > 0) + if err != nil { + return nil, nil, nil, err + } + return e.exportOCILayout(ctx, src, buildInfo, resolvedAnnotations, attests) + } + ctx, done, err := leaseutil.WithLease(ctx, e.opt.LeaseManager, leaseutil.MakeTemporary) if err != nil { return nil, nil, nil, err @@ -295,6 +310,123 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source return resp, nil, nil, nil } +func (e *imageExporterInstance) exportOCILayout(ctx context.Context, src *exporter.Source, buildInfo exporter.ExportBuildInfo, annotations *containerimage.Annotations, attestations []exporter.Attestation) (_ map[string]string, _ exporter.FinalizeFunc, _ exporter.DescriptorReference, err error) { + ctx, done, err := leaseutil.WithLease(ctx, e.opt.LeaseManager, leaseutil.MakeTemporary) + if err != nil { + return nil, nil, nil, err + } + defer done(context.WithoutCancel(ctx)) + + var ref cache.ImmutableRef + if src.Ref != nil { + ref = src.Ref + } else if len(src.Refs) == 1 { + for _, r := range src.Refs { + ref = r + } + } + if ref == nil { + return nil, nil, nil, errors.New("OCI layout export requires exactly one ref") + } + + desc, err := e.opt.ImageWriter.CommitOCILayout(ctx, ref, buildInfo.SessionID, annotations) + if err != nil { + return nil, nil, nil, err + } + if n, ok := src.Metadata["image.name"]; e.opts.ImageName == "*" && ok { + e.opts.ImageName = string(n) + } + if len(attestations) > 0 { + subjects, err := containerimage.DefaultArtifactSubjects(e.opts.ImageName, desc.Digest) + if err != nil { + return nil, nil, nil, err + } + desc, err = e.opt.ImageWriter.CommitAttestationsForDescriptor(ctx, &e.opts, *desc, buildInfo.SessionID, attestations, subjects, annotations) + if err != nil { + return nil, nil, nil, err + } + } + + if desc.Annotations == nil { + desc.Annotations = map[string]string{} + } + if _, ok := desc.Annotations[ocispecs.AnnotationCreated]; !ok { + tm := time.Now() + if e.opts.Epoch != nil { + tm = *e.opts.Epoch + } + desc.Annotations[ocispecs.AnnotationCreated] = tm.UTC().Format(time.RFC3339) + } + + resp := make(map[string]string) + resp[exptypes.ExporterImageDigestKey] = desc.Digest.String() + if v, ok := desc.Annotations[exptypes.ExporterConfigDigestKey]; ok { + resp[exptypes.ExporterImageConfigDigestKey] = v + delete(desc.Annotations, exptypes.ExporterConfigDigestKey) + } + + dtdesc, err := json.Marshal(desc) + if err != nil { + return nil, nil, nil, err + } + resp[exptypes.ExporterImageDescriptorKey] = base64.StdEncoding.EncodeToString(dtdesc) + + names, err := normalizedNames(e.opts.ImageName) + if err != nil { + return nil, nil, nil, err + } + if len(names) != 0 { + resp[exptypes.ExporterImageNameKey] = strings.Join(names, ",") + } + + expOpts := []archiveexporter.ExportOpt{ + archiveexporter.WithManifest(*desc, names...), + archiveexporter.WithAllPlatforms(), + archiveexporter.WithSkipDockerManifest(), + } + + timeoutCtx, cancel := context.WithCancelCause(ctx) + timeoutCtx, _ = context.WithTimeoutCause(timeoutCtx, 5*time.Second, errors.WithStack(context.DeadlineExceeded)) //nolint:govet + defer func() { cancel(errors.WithStack(context.Canceled)) }() + + caller, err := e.opt.SessionManager.Get(timeoutCtx, buildInfo.SessionID, false) + if err != nil { + return nil, nil, nil, err + } + + cs := e.opt.ImageWriter.ContentStore() + if e.tar { + w, err := filesync.CopyFileWriter(ctx, resp, e.id, caller) + if err != nil { + return nil, nil, nil, err + } + + report := progress.OneOff(ctx, "sending tarball") + if err := archiveexporter.Export(ctx, cs, w, expOpts...); err != nil { + w.Close() + if grpcerrors.Code(err) == codes.AlreadyExists { + return resp, nil, nil, report(nil) + } + return nil, nil, nil, report(err) + } + err = w.Close() + if grpcerrors.Code(err) == codes.AlreadyExists { + return resp, nil, nil, report(nil) + } + if err != nil { + return nil, nil, nil, report(err) + } + report(nil) + } else { + store := sessioncontent.NewCallerStore(caller, "export") + if err := contentutil.CopyChain(ctx, store, cs, *desc); err != nil { + return nil, nil, nil, err + } + } + + return resp, nil, nil, nil +} + func normalizedNames(name string) ([]string, error) { if name == "" { return nil, nil diff --git a/frontend/artifact/artifact.go b/frontend/artifact/artifact.go new file mode 100644 index 000000000000..f1f023ae8de6 --- /dev/null +++ b/frontend/artifact/artifact.go @@ -0,0 +1,239 @@ +package artifact + +import ( + "context" + "encoding/json" + "path" + + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + gateway "github.com/moby/buildkit/frontend/gateway/client" + digest "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +const ( + Name = "artifact.v0" + InputKey = "input" + + digestChunkSize = 4 << 20 +) + +func Build(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + inputState, err := resolveInputState(ctx, c) + if err != nil { + return nil, err + } + + opts := c.BuildOpts().Opts + artifactType := opts[exptypes.ExporterArtifactTypeKey] + configMediaType := opts[exptypes.ExporterArtifactConfigTypeKey] + + var layers []exptypes.ArtifactLayer + if err := json.Unmarshal([]byte(opts[exptypes.ExporterArtifactLayersKey]), &layers); err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to parse layers metadata") + } + layers, err = normalizeAndValidateArtifactMetadata(configMediaType, layers) + if err != nil { + return nil, err + } + + inputDef, err := inputState.Marshal(ctx) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to marshal input state") + } + inputRes, err := c.Solve(ctx, gateway.SolveRequest{ + Definition: inputDef.ToPB(), + }) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to solve input state") + } + inputRef, err := inputRes.SingleRef() + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to resolve input ref") + } + + configContent := []byte("{}") + configDigest := digest.FromBytes(configContent) + configDesc := ocispecs.Descriptor{ + MediaType: configMediaType, + Digest: configDigest, + Size: int64(len(configContent)), + } + + st := llb.Scratch(). + File(llb.Mkdir("/blobs/sha256", 0755, llb.WithParents(true))). + File(llb.Mkfile("/blobs/sha256/"+configDigest.Encoded(), 0644, configContent)) + + layerDescs := make([]ocispecs.Descriptor, 0, len(layers)) + for _, layer := range layers { + stat, err := inputRef.StatFile(ctx, gateway.StatRequest{Path: layer.Path}) + if err != nil { + return nil, errors.Wrapf(err, "artifact frontend: failed to stat layer file %s relative to input root", layer.Path) + } + if stat.IsDir() { + return nil, errors.Errorf("artifact frontend: layer path %s must be a file", layer.Path) + } + + dgst, err := digestFile(ctx, inputRef, layer.Path, stat.Size) + if err != nil { + return nil, err + } + desc := ocispecs.Descriptor{ + MediaType: layer.MediaType, + Digest: dgst, + Size: stat.Size, + Annotations: layer.Annotations, + } + layerDescs = append(layerDescs, desc) + + st = st.File(llb.Copy(inputState, layer.Path, "/blobs/sha256/"+dgst.Encoded(), &llb.CopyInfo{ + FollowSymlinks: true, + })) + } + + manifest := ocispecs.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ocispecs.MediaTypeImageManifest, + Config: configDesc, + Layers: layerDescs, + } + if artifactType != "" { + manifest.ArtifactType = artifactType + } + + manifestJSON, err := json.Marshal(manifest) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to marshal manifest") + } + manifestDigest := digest.FromBytes(manifestJSON) + + index := ocispecs.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + Manifests: []ocispecs.Descriptor{{ + MediaType: ocispecs.MediaTypeImageManifest, + Digest: manifestDigest, + Size: int64(len(manifestJSON)), + ArtifactType: artifactType, + }}, + } + indexJSON, err := json.Marshal(index) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to marshal index") + } + + layoutJSON, err := json.Marshal(ocispecs.ImageLayout{Version: ocispecs.ImageLayoutVersion}) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to marshal oci-layout") + } + + st = st.File(llb.Mkfile("/blobs/sha256/"+manifestDigest.Encoded(), 0644, manifestJSON)) + st = st.File(llb.Mkfile("/index.json", 0644, indexJSON)) + st = st.File(llb.Mkfile("/oci-layout", 0644, layoutJSON)) + + def, err := st.Marshal(ctx, llb.WithCustomName("[artifact] assembling OCI layout")) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to marshal OCI layout LLB") + } + + res, err := c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to solve OCI layout assembly") + } + ref, err := res.SingleRef() + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: solve returned no ref") + } + + out := gateway.NewResult() + out.SetRef(ref) + return out, nil +} + +func resolveInputState(ctx context.Context, c gateway.Client) (llb.State, error) { + inputs, err := c.Inputs(ctx) + if err != nil { + return llb.State{}, errors.Wrap(err, "artifact frontend: failed to load inputs") + } + if st, ok := inputs[InputKey]; ok { + return st, nil + } + if len(inputs) == 1 { + for _, st := range inputs { + return st, nil + } + } + return llb.State{}, errors.New("artifact frontend: missing input state") +} + +func normalizeAndValidateArtifactMetadata(configMediaType string, layers []exptypes.ArtifactLayer) ([]exptypes.ArtifactLayer, error) { + if configMediaType == "" { + return nil, errors.New("artifact frontend: config descriptor mediaType is required") + } + normalized := make([]exptypes.ArtifactLayer, len(layers)) + for i, layer := range layers { + cleanedPath, err := normalizeLayerPath(layer.Path) + if err != nil { + return nil, errors.Wrapf(err, "artifact frontend: invalid layer %d path", i) + } + if layer.MediaType == "" { + return nil, errors.Errorf("artifact frontend: layer %d descriptor mediaType is required", i) + } + layer.Path = cleanedPath + normalized[i] = layer + } + return normalized, nil +} + +func normalizeLayerPath(p string) (string, error) { + if p == "" { + return "", errors.New("path is required") + } + cleaned := path.Clean(p) + switch cleaned { + case ".", "/": + return "", errors.New("path must reference a file within the artifact input root") + case "..": + return "", errors.New("path must be relative to the artifact input root") + } + if cleaned[0] == '/' || len(cleaned) >= 3 && cleaned[:3] == "../" { + return "", errors.New("path must be relative to the artifact input root") + } + return cleaned, nil +} + +func digestFile(ctx context.Context, ref gateway.Reference, path string, size int64) (digest.Digest, error) { + digester := digest.Canonical.Digester() + hash := digester.Hash() + + for offset := int64(0); offset < size; { + length := digestChunkSize + if remaining := size - offset; remaining < int64(length) { + length = int(remaining) + } + + dt, err := ref.ReadFile(ctx, gateway.ReadRequest{ + Filename: path, + Range: &gateway.FileRange{ + Offset: int(offset), + Length: length, + }, + }) + if err != nil { + return "", errors.Wrapf(err, "artifact frontend: failed to read layer file %s", path) + } + if len(dt) == 0 && length > 0 { + return "", errors.Errorf("artifact frontend: short read while hashing %s", path) + } + if _, err := hash.Write(dt); err != nil { + return "", errors.Wrapf(err, "artifact frontend: failed to hash layer file %s", path) + } + offset += int64(len(dt)) + } + + return digester.Digest(), nil +} diff --git a/frontend/artifact/artifact_test.go b/frontend/artifact/artifact_test.go new file mode 100644 index 000000000000..1ec028018a85 --- /dev/null +++ b/frontend/artifact/artifact_test.go @@ -0,0 +1,63 @@ +package artifact + +import ( + "testing" + + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/stretchr/testify/require" +) + +func TestNormalizeAndValidateArtifactMetadata(t *testing.T) { + t.Run("normalizes relative paths", func(t *testing.T) { + layers, err := normalizeAndValidateArtifactMetadata("application/vnd.test.config.v1+json", []exptypes.ArtifactLayer{{ + Path: "./subdir/../artifact.txt", + MediaType: "application/vnd.test.layer.v1", + }}) + require.NoError(t, err) + require.Len(t, layers, 1) + require.Equal(t, "artifact.txt", layers[0].Path) + }) + + t.Run("rejects missing config media type", func(t *testing.T) { + _, err := normalizeAndValidateArtifactMetadata("", nil) + require.EqualError(t, err, "artifact frontend: config descriptor mediaType is required") + }) + + t.Run("rejects missing layer path", func(t *testing.T) { + _, err := normalizeAndValidateArtifactMetadata("application/vnd.test.config.v1+json", []exptypes.ArtifactLayer{{ + MediaType: "application/vnd.test.layer.v1", + }}) + require.EqualError(t, err, "artifact frontend: invalid layer 0 path: path is required") + }) + + t.Run("rejects absolute paths", func(t *testing.T) { + _, err := normalizeAndValidateArtifactMetadata("application/vnd.test.config.v1+json", []exptypes.ArtifactLayer{{ + Path: "/artifact.txt", + MediaType: "application/vnd.test.layer.v1", + }}) + require.EqualError(t, err, "artifact frontend: invalid layer 0 path: path must be relative to the artifact input root") + }) + + t.Run("rejects parent traversal", func(t *testing.T) { + _, err := normalizeAndValidateArtifactMetadata("application/vnd.test.config.v1+json", []exptypes.ArtifactLayer{{ + Path: "../artifact.txt", + MediaType: "application/vnd.test.layer.v1", + }}) + require.EqualError(t, err, "artifact frontend: invalid layer 0 path: path must be relative to the artifact input root") + }) + + t.Run("rejects root path", func(t *testing.T) { + _, err := normalizeAndValidateArtifactMetadata("application/vnd.test.config.v1+json", []exptypes.ArtifactLayer{{ + Path: ".", + MediaType: "application/vnd.test.layer.v1", + }}) + require.EqualError(t, err, "artifact frontend: invalid layer 0 path: path must reference a file within the artifact input root") + }) + + t.Run("rejects missing layer media type", func(t *testing.T) { + _, err := normalizeAndValidateArtifactMetadata("application/vnd.test.config.v1+json", []exptypes.ArtifactLayer{{ + Path: "artifact.txt", + }}) + require.EqualError(t, err, "artifact frontend: layer 0 descriptor mediaType is required") + }) +} diff --git a/solver/llbsolver/proc/artifact.go b/solver/llbsolver/proc/artifact.go new file mode 100644 index 000000000000..f490a96b1d03 --- /dev/null +++ b/solver/llbsolver/proc/artifact.go @@ -0,0 +1,149 @@ +package proc + +import ( + "context" + "encoding/json" + "maps" + + "github.com/moby/buildkit/executor/resources" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/frontend" + artifactfrontend "github.com/moby/buildkit/frontend/artifact" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/llbsolver" + solverpb "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/tracing" + "github.com/pkg/errors" +) + +func ArtifactProcessor() llbsolver.Processor { + return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job, usage *resources.SysSampler) (_ *llbsolver.Result, err error) { + if _, ok := res.Metadata[exptypes.ExporterArtifactKey]; !ok { + return res, nil + } + + span, ctx := tracing.StartSpan(ctx, "create artifact OCI layout") + defer span.End() + + if len(res.Refs) > 1 { + return nil, errors.New("artifact processor requires exactly one ref") + } + + ref, ok := res.FindRef("") + if !ok || ref == nil { + return nil, errors.New("artifact processor requires exactly one ref") + } + + postRes, err := s.SolvePostprocessor(ctx, j, frontend.SolveRequest{ + Frontend: artifactfrontend.Name, + FrontendOpt: map[string]string{ + exptypes.ExporterArtifactTypeKey: string(res.Metadata[exptypes.ExporterArtifactTypeKey]), + exptypes.ExporterArtifactConfigTypeKey: string(res.Metadata[exptypes.ExporterArtifactConfigTypeKey]), + exptypes.ExporterArtifactLayersKey: string(res.Metadata[exptypes.ExporterArtifactLayersKey]), + }, + FrontendInputs: map[string]*solverpb.Definition{ + artifactfrontend.InputKey: ref.Definition(), + }, + }) + if err != nil { + return nil, errors.Wrap(err, "artifact processor: failed to run artifact frontend") + } + + if err := updateArtifactProvenance(res, postRes); err != nil { + return nil, err + } + postMeta := maps.Clone(res.Metadata) + if err := normalizeArtifactResultMetadata(postMeta); err != nil { + return nil, err + } + postMeta[exptypes.ExporterOCILayoutKey] = []byte("true") + delete(postMeta, exptypes.ExporterArtifactKey) + delete(postMeta, exptypes.ExporterArtifactTypeKey) + delete(postMeta, exptypes.ExporterArtifactConfigTypeKey) + delete(postMeta, exptypes.ExporterArtifactLayersKey) + if postRes.Result.Metadata == nil { + postRes.Result.Metadata = map[string][]byte{} + } + maps.Copy(postRes.Result.Metadata, postMeta) + if len(res.Attestations) > 0 { + if postRes.Result.Attestations == nil { + postRes.Result.Attestations = map[string][]frontend.Attestation{} + } + maps.Copy(postRes.Result.Attestations, res.Attestations) + } + + res.Result = postRes.Result + + if err := ref.Release(context.WithoutCancel(ctx)); err != nil { + return nil, errors.Wrap(err, "artifact processor: failed to release original result ref") + } + + return res, nil + } +} + +func updateArtifactProvenance(res *llbsolver.Result, postRes *llbsolver.Result) error { + if postRes == nil || postRes.Provenance == nil { + if res.Provenance != nil { + res.Provenance.Refs = nil + } + return nil + } + if res.Provenance == nil { + res.Provenance = postRes.Provenance + res.Provenance.Refs = nil + return nil + } + + oldProv := res.Provenance.Ref + newProv := postRes.Provenance.Ref + if newProv == nil { + res.Provenance.Ref = oldProv + res.Provenance.Refs = nil + return nil + } + if oldProv != nil { + postprocess := newProv.Parameters() + if err := newProv.Merge(oldProv); err != nil { + return errors.Wrap(err, "artifact processor: failed to merge provenance") + } + newProv.AddPostprocess(postprocess) + newProv.Frontend = oldProv.Frontend + newProv.Args = maps.Clone(oldProv.Args) + } + + res.Provenance = postRes.Provenance + res.Provenance.Ref = newProv + res.Provenance.Refs = nil + return nil +} + +func normalizeArtifactResultMetadata(meta map[string][]byte) error { + if meta == nil { + return nil + } + + platformsBytes, ok := meta[exptypes.ExporterPlatformsKey] + if !ok { + return nil + } + + var platforms exptypes.Platforms + if err := json.Unmarshal(platformsBytes, &platforms); err != nil { + return errors.Wrap(err, "artifact processor: failed to parse platforms metadata") + } + if len(platforms.Platforms) != 1 { + return errors.Errorf("artifact processor: expected exactly one platform mapping, got %d", len(platforms.Platforms)) + } + + platformID := platforms.Platforms[0].ID + for _, key := range exptypes.KnownRefMetadataKeys { + if _, ok := meta[key]; ok { + continue + } + if v, ok := meta[key+"/"+platformID]; ok { + meta[key] = v + } + } + return nil +} diff --git a/solver/llbsolver/proc/artifact_test.go b/solver/llbsolver/proc/artifact_test.go new file mode 100644 index 000000000000..311ed3f3065c --- /dev/null +++ b/solver/llbsolver/proc/artifact_test.go @@ -0,0 +1,69 @@ +package proc + +import ( + "encoding/json" + "testing" + + "github.com/containerd/platforms" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/frontend" + artifactfrontend "github.com/moby/buildkit/frontend/artifact" + "github.com/moby/buildkit/solver/llbsolver" + "github.com/moby/buildkit/solver/llbsolver/provenance" + "github.com/stretchr/testify/require" +) + +func TestUpdateArtifactProvenanceAddsPostprocess(t *testing.T) { + res := &llbsolver.Result{ + Result: &frontend.Result{}, + Provenance: &provenance.Result{ + Ref: &provenance.Capture{ + Frontend: "dockerfile.v0", + Args: map[string]string{ + "target": "release", + }, + }, + }, + } + + postRes := &llbsolver.Result{ + Result: &frontend.Result{}, + Provenance: &provenance.Result{ + Ref: &provenance.Capture{ + Frontend: artifactfrontend.Name, + Args: map[string]string{ + "type": "artifact", + }, + }, + }, + } + + require.NoError(t, updateArtifactProvenance(res, postRes)) + require.NotNil(t, res.Provenance) + require.NotNil(t, res.Provenance.Ref) + require.Equal(t, "dockerfile.v0", res.Provenance.Ref.Frontend) + require.Equal(t, "release", res.Provenance.Ref.Args["target"]) + require.Len(t, res.Provenance.Ref.Postprocess, 1) + require.Equal(t, artifactfrontend.Name, res.Provenance.Ref.Postprocess[0].Frontend) + require.Equal(t, "artifact", res.Provenance.Ref.Postprocess[0].Args["type"]) +} + +func TestNormalizeArtifactResultMetadataPreservesSinglePlatformMapping(t *testing.T) { + ps := exptypes.Platforms{ + Platforms: []exptypes.Platform{{ + ID: platforms.Format(platforms.MustParse("linux/arm64")), + Platform: platforms.MustParse("linux/arm64"), + }}, + } + psJSON, err := json.Marshal(ps) + require.NoError(t, err) + + meta := map[string][]byte{ + exptypes.ExporterPlatformsKey: psJSON, + exptypes.ExporterImageConfigKey + "/" + ps.Platforms[0].ID: []byte("config"), + } + + require.NoError(t, normalizeArtifactResultMetadata(meta)) + require.Equal(t, []byte("config"), meta[exptypes.ExporterImageConfigKey]) + require.Equal(t, psJSON, meta[exptypes.ExporterPlatformsKey]) +} diff --git a/solver/llbsolver/provenance/capture.go b/solver/llbsolver/provenance/capture.go index 800304409681..5dfc44c1b292 100644 --- a/solver/llbsolver/provenance/capture.go +++ b/solver/llbsolver/provenance/capture.go @@ -2,6 +2,7 @@ package provenance import ( "cmp" + "maps" "slices" distreference "github.com/distribution/reference" @@ -17,6 +18,7 @@ type Result = result.Result[*Capture] type Capture struct { Frontend string Args map[string]string + Postprocess []provenancetypes.Parameters Sources provenancetypes.Sources Secrets []provenancetypes.Secret SSH []provenancetypes.SSH @@ -50,6 +52,9 @@ func (c *Capture) Merge(c2 *Capture) error { for _, s := range c2.SSH { c.AddSSH(s) } + for _, p := range c2.Postprocess { + c.AddPostprocess(p) + } if c2.NetworkAccess { c.NetworkAccess = true } @@ -197,6 +202,38 @@ func (c *Capture) AddSSH(s provenancetypes.SSH) { c.SSH = append(c.SSH, s) } +func (c *Capture) Parameters() provenancetypes.Parameters { + p := provenancetypes.Parameters{ + Frontend: c.Frontend, + Args: maps.Clone(c.Args), + } + for _, s := range c.Secrets { + s2 := s + p.Secrets = append(p.Secrets, &s2) + } + for _, s := range c.SSH { + s2 := s + p.SSH = append(p.SSH, &s2) + } + for _, s := range c.Sources.Local { + s2 := s + p.Locals = append(p.Locals, &s2) + } + return p +} + +func (c *Capture) AddPostprocess(p provenancetypes.Parameters) { + if p.Frontend == "" { + return + } + for _, existing := range c.Postprocess { + if equalParameters(existing, p) { + return + } + } + c.Postprocess = append(c.Postprocess, cloneParameters(p)) +} + func (c *Capture) AddSamples(dgst digest.Digest, samples *resourcestypes.Samples) { if c.Samples == nil { c.Samples = map[digest.Digest]*resourcestypes.Samples{} @@ -204,6 +241,84 @@ func (c *Capture) AddSamples(dgst digest.Digest, samples *resourcestypes.Samples c.Samples[dgst] = samples } +func cloneParameters(p provenancetypes.Parameters) provenancetypes.Parameters { + out := provenancetypes.Parameters{ + Frontend: p.Frontend, + Args: maps.Clone(p.Args), + } + for _, s := range p.Secrets { + if s == nil { + out.Secrets = append(out.Secrets, nil) + continue + } + s2 := *s + out.Secrets = append(out.Secrets, &s2) + } + for _, s := range p.SSH { + if s == nil { + out.SSH = append(out.SSH, nil) + continue + } + s2 := *s + out.SSH = append(out.SSH, &s2) + } + for _, l := range p.Locals { + if l == nil { + out.Locals = append(out.Locals, nil) + continue + } + l2 := *l + out.Locals = append(out.Locals, &l2) + } + return out +} + +func equalParameters(a, b provenancetypes.Parameters) bool { + if a.Frontend != b.Frontend || !maps.Equal(a.Args, b.Args) { + return false + } + if len(a.Secrets) != len(b.Secrets) || len(a.SSH) != len(b.SSH) || len(a.Locals) != len(b.Locals) { + return false + } + for i := range a.Secrets { + if !equalSecret(a.Secrets[i], b.Secrets[i]) { + return false + } + } + for i := range a.SSH { + if !equalSSH(a.SSH[i], b.SSH[i]) { + return false + } + } + for i := range a.Locals { + if !equalLocal(a.Locals[i], b.Locals[i]) { + return false + } + } + return true +} + +func equalSecret(a, b *provenancetypes.Secret) bool { + if a == nil || b == nil { + return a == b + } + return a.ID == b.ID && a.Optional == b.Optional +} + +func equalSSH(a, b *provenancetypes.SSH) bool { + if a == nil || b == nil { + return a == b + } + return a.ID == b.ID && a.Optional == b.Optional +} + +func equalLocal(a, b *provenancetypes.LocalSource) bool { + if a == nil || b == nil { + return a == b + } + return a.Name == b.Name +} + func parseRefName(s string) (distreference.Named, string, error) { ref, err := distreference.ParseNormalizedNamed(s) if err != nil { diff --git a/solver/llbsolver/provenance/predicate.go b/solver/llbsolver/provenance/predicate.go index d82d04cb0474..f3ca2cf0d48b 100644 --- a/solver/llbsolver/provenance/predicate.go +++ b/solver/llbsolver/provenance/predicate.go @@ -188,6 +188,9 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error) internal := provenancetypes.ProvenanceInternalParametersSLSA1{} internal.BuilderPlatform = platforms.Format(platforms.Normalize(platforms.DefaultSpec())) + if len(c.Postprocess) > 0 { + internal.Postprocess = cloneParametersSlice(c.Postprocess) + } req := provenancetypes.Parameters{} req.Frontend = c.Frontend @@ -245,6 +248,17 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error) return pr, nil } +func cloneParametersSlice(src []provenancetypes.Parameters) []provenancetypes.Parameters { + if len(src) == 0 { + return nil + } + out := make([]provenancetypes.Parameters, 0, len(src)) + for _, p := range src { + out = append(out, cloneParameters(p)) + } + return out +} + func FilterArgs(m map[string]string) map[string]string { var hostSpecificArgs = map[string]struct{}{ "cgroup-parent": {}, diff --git a/solver/llbsolver/provenance/types/types.go b/solver/llbsolver/provenance/types/types.go index c077f07158ca..c2202e920b25 100644 --- a/solver/llbsolver/provenance/types/types.go +++ b/solver/llbsolver/provenance/types/types.go @@ -160,6 +160,7 @@ type ProvenanceConfigSourceSLSA1 struct { type ProvenanceInternalParametersSLSA1 struct { BuildConfig *BuildConfig `json:"buildConfig,omitempty"` BuilderPlatform string `json:"builderPlatform"` + Postprocess []Parameters `json:"postprocess,omitempty"` ProvenanceCustomEnv } @@ -182,7 +183,8 @@ type Parameters struct { } type Environment struct { - Platform string `json:"platform"` + Platform string `json:"platform"` + Postprocess []Parameters `json:"postprocess,omitempty"` ProvenanceCustomEnv } @@ -246,6 +248,7 @@ func (p *ProvenancePredicateSLSA1) ConvertToSLSA02() *ProvenancePredicateSLSA02 Parameters: p.BuildDefinition.ExternalParameters.Request, Environment: Environment{ Platform: p.BuildDefinition.InternalParameters.BuilderPlatform, + Postprocess: p.BuildDefinition.InternalParameters.Postprocess, ProvenanceCustomEnv: p.BuildDefinition.InternalParameters.ProvenanceCustomEnv, }, }, @@ -280,6 +283,7 @@ func (p *ProvenancePredicateSLSA02) ConvertToSLSA1() *ProvenancePredicateSLSA1 { InternalParameters: ProvenanceInternalParametersSLSA1{ BuildConfig: p.BuildConfig, BuilderPlatform: p.Invocation.Environment.Platform, + Postprocess: p.Invocation.Environment.Postprocess, ProvenanceCustomEnv: p.Invocation.Environment.ProvenanceCustomEnv, }, } diff --git a/solver/llbsolver/provenance/types/types_test.go b/solver/llbsolver/provenance/types/types_test.go index 5da999944775..30e22eeba53b 100644 --- a/solver/llbsolver/provenance/types/types_test.go +++ b/solver/llbsolver/provenance/types/types_test.go @@ -16,6 +16,12 @@ func TestMarsalBuildDefinitionSLSA1(t *testing.T) { }, "internalParameters": { "builderPlatform": "linux/amd64", + "postprocess": [{ + "frontend": "artifact.v0", + "args": { + "type": "oci" + } + }], "foo": "bar", "abc": 123, "def": {"one": 1} @@ -28,6 +34,9 @@ func TestMarsalBuildDefinitionSLSA1(t *testing.T) { require.Equal(t, "btype1", def.BuildType) require.Equal(t, "linux/amd64", def.InternalParameters.BuilderPlatform) + require.Len(t, def.InternalParameters.Postprocess, 1) + require.Equal(t, "artifact.v0", def.InternalParameters.Postprocess[0].Frontend) + require.Equal(t, "oci", def.InternalParameters.Postprocess[0].Args["type"]) require.Equal(t, "bar", def.InternalParameters.ProvenanceCustomEnv["foo"]) require.InEpsilon(t, float64(123), def.InternalParameters.ProvenanceCustomEnv["abc"], 0.001) require.Equal(t, map[string]any{"one": float64(1)}, def.InternalParameters.ProvenanceCustomEnv["def"]) @@ -48,6 +57,9 @@ func TestMarshalInvocation(t *testing.T) { }, "environment": { "platform": "linux/amd64", + "postprocess": [{ + "frontend": "artifact.v0" + }], "buildkit": "v0.10.3", "custom": { "foo": "bar" @@ -63,6 +75,8 @@ func TestMarshalInvocation(t *testing.T) { require.Equal(t, "git+https://github.com/example/repo.git", inv.ConfigSource.URI) require.Equal(t, "dockerfile.v0", inv.Parameters.Frontend) require.Equal(t, "linux/amd64", inv.Environment.Platform) + require.Len(t, inv.Environment.Postprocess, 1) + require.Equal(t, "artifact.v0", inv.Environment.Postprocess[0].Frontend) require.Equal(t, "v0.10.3", inv.Environment.ProvenanceCustomEnv["buildkit"]) require.Equal(t, "bar", inv.Environment.ProvenanceCustomEnv["custom"].(map[string]any)["foo"]) require.Equal(t, []any{float64(1), float64(2), float64(3)}, inv.Environment.ProvenanceCustomEnv["bar"]) diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index c1f6317fe513..eab183b178e9 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -152,6 +152,32 @@ func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge { return s.bridge(b) } +// SolvePostprocessor runs an additional solve through the frontend pipeline so +// the converted result is tracked like a regular frontend solve before export. +func (s *Solver) SolvePostprocessor(ctx context.Context, j *solver.Job, req frontend.SolveRequest) (*Result, error) { + br := s.bridge(j) + + res, err := br.Solve(ctx, req, j.SessionID) + if err != nil { + return nil, err + } + + eg, ctx2 := errgroup.WithContext(ctx) + res.EachRef(func(ref solver.ResultProxy) error { + eg.Go(func() error { + _, err := ref.Result(ctx2) + return err + }) + return nil + }) + if err := eg.Wait(); err != nil { + return nil, err + } + + return addProvenanceToResult(res, br) +} + + func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req frontend.SolveRequest, exp ExporterRequest, ent []entitlements.Entitlement, post []Processor, internal bool, srcPol *spb.Policy, policySession string) (_ *client.SolveResponse, err error) { j, err := s.solver.NewJob(id) if err != nil { diff --git a/util/push/push.go b/util/push/push.go index 77788da20cb4..a336f194e683 100644 --- a/util/push/push.go +++ b/util/push/push.go @@ -255,7 +255,7 @@ func childrenHandler(provider content.Provider) images.HandlerFunc { // childless data types. return nil, nil default: - bklog.G(ctx).Warnf("encountered unknown type %v; children may not be fetched", desc.MediaType) + bklog.G(ctx).Debugf("encountered unknown type %v; children may not be fetched", desc.MediaType) } return descs, nil