diff --git a/client/client_test.go b/client/client_test.go index 33344b601d62..b658896f2b75 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -208,6 +208,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){ testPullWithLayerLimit, testExportAnnotations, testExportAnnotationsMediaTypes, + testOCIArtifactPostprocessor, testExportAttestationsOCIArtifact, testExportAttestationsImageManifest, testExportedImageLabels, @@ -10014,6 +10015,411 @@ func testExportAnnotationsMediaTypes(t *testing.T, sb integration.Sandbox) { require.Equal(t, ocispecs.MediaTypeImageIndex, imgs2.Index.MediaType) } +func testOCIArtifactPostprocessor(t *testing.T, sb integration.Sandbox) { + c, err := New(sb.Context(), sb.Address()) + require.NoError(t, err) + defer c.Close() + + const ( + artifactType = "application/vnd.buildkit.test.artifact.v1" + configMediaType = "application/vnd.buildkit.test.config.v1+json" + layerMediaType = "application/vnd.buildkit.test.layer.v1" + artifactFile = "artifact.txt" + ) + + artifactPayload := []byte("artifact payload") + expectedConfigDigest := digest.FromBytes([]byte("{}")) + + artifactFrontend := func(configType, layerType string) gateway.BuildFunc { + return func(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + st := integration.UnixOrWindows(llb.Scratch(), llb.Image("nanoserver")).File( + llb.Mkfile(artifactFile, 0600, artifactPayload), + ) + def, err := st.Marshal(ctx) + if err != nil { + return nil, err + } + r, err := c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, err + } + ref, err := r.SingleRef() + if err != nil { + return nil, err + } + + layers, err := json.Marshal([]exptypes.ArtifactLayer{{ + Path: artifactFile, + MediaType: layerType, + Annotations: map[string]string{ + ocispecs.AnnotationTitle: artifactFile, + }, + }}) + if err != nil { + return nil, err + } + + res := gateway.NewResult() + res.SetRef(ref) + res.AddMeta(exptypes.ExporterArtifactKey, []byte("true")) + res.AddMeta(exptypes.ExporterArtifactTypeKey, []byte(artifactType)) + res.AddMeta(exptypes.ExporterArtifactConfigTypeKey, []byte(configType)) + res.AddMeta(exptypes.ExporterArtifactLayersKey, layers) + return res, nil + } + } + + decodeDescriptor := func(t *testing.T, encoded string) ocispecs.Descriptor { + t.Helper() + + dt, err := base64.StdEncoding.DecodeString(encoded) + require.NoError(t, err) + + var desc ocispecs.Descriptor + require.NoError(t, json.Unmarshal(dt, &desc)) + return desc + } + + type provenanceStatement struct { + intoto.StatementHeader + Predicate provenancetypes.ProvenancePredicateSLSA1 `json:"predicate"` + } + + findArtifactDescriptors := func(t *testing.T, manifests []ocispecs.Descriptor) (ocispecs.Descriptor, ocispecs.Descriptor) { + t.Helper() + + var artifactDesc, attestationDesc ocispecs.Descriptor + for _, desc := range manifests { + switch { + case desc.Platform == nil: + artifactDesc = desc + case platforms.Format(*desc.Platform) == "unknown/unknown": + attestationDesc = desc + } + } + + require.NotEmpty(t, artifactDesc.Digest) + require.NotEmpty(t, attestationDesc.Digest) + return artifactDesc, attestationDesc + } + + t.Run("local", func(t *testing.T) { + dir := t.TempDir() + _, err := c.Build(sb.Context(), SolveOpt{ + Exports: []ExportEntry{{ + Type: ExporterLocal, + OutputDir: dir, + }}, + }, "", artifactFrontend(configMediaType, layerMediaType), nil) + require.NoError(t, err) + + _, err = os.Stat(filepath.Join(dir, artifactFile)) + require.ErrorIs(t, err, os.ErrNotExist) + + layoutData, err := os.ReadFile(filepath.Join(dir, ocispecs.ImageLayoutFile)) + require.NoError(t, err) + var layout ocispecs.ImageLayout + require.NoError(t, json.Unmarshal(layoutData, &layout)) + require.Equal(t, ocispecs.ImageLayoutVersion, layout.Version) + + indexData, err := os.ReadFile(filepath.Join(dir, ocispecs.ImageIndexFile)) + require.NoError(t, err) + var index ocispecs.Index + require.NoError(t, json.Unmarshal(indexData, &index)) + require.Len(t, index.Manifests, 1) + require.Equal(t, artifactType, index.Manifests[0].ArtifactType) + require.Equal(t, ocispecs.MediaTypeImageManifest, index.Manifests[0].MediaType) + + manifestData, err := os.ReadFile(filepath.Join(dir, ocispecs.ImageBlobsDir, "sha256", index.Manifests[0].Digest.Encoded())) + require.NoError(t, err) + var manifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(manifestData, &manifest)) + require.Equal(t, artifactType, manifest.ArtifactType) + require.Equal(t, configMediaType, manifest.Config.MediaType) + require.Equal(t, expectedConfigDigest, manifest.Config.Digest) + require.Len(t, manifest.Layers, 1) + require.Equal(t, layerMediaType, manifest.Layers[0].MediaType) + require.Equal(t, artifactFile, manifest.Layers[0].Annotations[ocispecs.AnnotationTitle]) + + layerData, err := os.ReadFile(filepath.Join(dir, ocispecs.ImageBlobsDir, "sha256", manifest.Layers[0].Digest.Encoded())) + require.NoError(t, err) + require.Equal(t, artifactPayload, layerData) + }) + + t.Run("local-provenance", func(t *testing.T) { + dir := t.TempDir() + _, err := c.Build(sb.Context(), SolveOpt{ + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=max,version=v1", + }, + Exports: []ExportEntry{{ + Type: ExporterLocal, + OutputDir: dir, + }}, + }, "", artifactFrontend(configMediaType, layerMediaType), nil) + require.NoError(t, err) + + provDt, err := os.ReadFile(filepath.Join(dir, "provenance.json")) + require.NoError(t, err) + + var stmt provenanceStatement + require.NoError(t, json.Unmarshal(provDt, &stmt)) + require.Len(t, stmt.Predicate.BuildDefinition.InternalParameters.Postprocess, 1) + require.Equal(t, "artifact.v0", stmt.Predicate.BuildDefinition.InternalParameters.Postprocess[0].Frontend) + require.Equal(t, artifactType, stmt.Predicate.BuildDefinition.InternalParameters.Postprocess[0].Args[exptypes.ExporterArtifactTypeKey]) + }) + + t.Run("oci", func(t *testing.T) { + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter) + + dir := t.TempDir() + out := filepath.Join(dir, "artifact.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + resp, err := c.Build(sb.Context(), SolveOpt{ + Exports: []ExportEntry{{ + Type: ExporterOCI, + Output: fixedWriteCloser(outW), + }}, + }, "", artifactFrontend(configMediaType, layerMediaType), nil) + require.NoError(t, err) + + desc := decodeDescriptor(t, resp.ExporterResponse[exptypes.ExporterImageDescriptorKey]) + require.Equal(t, artifactType, desc.ArtifactType) + require.Equal(t, expectedConfigDigest.String(), resp.ExporterResponse[exptypes.ExporterImageConfigDigestKey]) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + + m, err := testutil.ReadTarToMap(dt, false) + require.NoError(t, err) + + var index ocispecs.Index + require.NoError(t, json.Unmarshal(m[ocispecs.ImageIndexFile].Data, &index)) + require.Len(t, index.Manifests, 1) + require.Equal(t, artifactType, index.Manifests[0].ArtifactType) + + var manifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+index.Manifests[0].Digest.Hex()].Data, &manifest)) + require.Equal(t, artifactType, manifest.ArtifactType) + require.Equal(t, configMediaType, manifest.Config.MediaType) + require.Equal(t, expectedConfigDigest, manifest.Config.Digest) + }) + + t.Run("oci-inline-provenance", func(t *testing.T) { + workers.CheckFeatureCompat(t, sb, workers.FeatureOCIExporter) + + dir := t.TempDir() + out := filepath.Join(dir, "artifact-with-provenance.tar") + outW, err := os.Create(out) + require.NoError(t, err) + + resp, err := c.Build(sb.Context(), SolveOpt{ + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=max,inline-only=true,version=v1", + }, + Exports: []ExportEntry{{ + Type: ExporterOCI, + Output: fixedWriteCloser(outW), + Attrs: map[string]string{ + "annotation-index.ai": "artifact index", + "annotation-index-descriptor.aid": "artifact index descriptor", + "annotation-manifest.am": "artifact manifest", + "annotation-manifest-descriptor.amd": "artifact manifest descriptor", + }, + }}, + }, "", artifactFrontend(configMediaType, layerMediaType), nil) + require.NoError(t, err) + + desc := decodeDescriptor(t, resp.ExporterResponse[exptypes.ExporterImageDescriptorKey]) + require.Equal(t, ocispecs.MediaTypeImageIndex, desc.MediaType) + require.Equal(t, expectedConfigDigest.String(), resp.ExporterResponse[exptypes.ExporterImageConfigDigestKey]) + + dt, err := os.ReadFile(out) + require.NoError(t, err) + + m, err := testutil.ReadTarToMap(dt, false) + require.NoError(t, err) + + var layout ocispecs.Index + require.NoError(t, json.Unmarshal(m[ocispecs.ImageIndexFile].Data, &layout)) + require.Len(t, layout.Manifests, 1) + require.Equal(t, "artifact index descriptor", layout.Manifests[0].Annotations["aid"]) + + var index ocispecs.Index + require.NoError(t, json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+layout.Manifests[0].Digest.Hex()].Data, &index)) + require.Equal(t, "artifact index", index.Annotations["ai"]) + require.Len(t, index.Manifests, 2) + + artifactDesc, attestationDesc := findArtifactDescriptors(t, index.Manifests) + require.Equal(t, artifactType, artifactDesc.ArtifactType) + require.Equal(t, "artifact manifest descriptor", artifactDesc.Annotations["amd"]) + require.Equal(t, "unknown/unknown", platforms.Format(*attestationDesc.Platform)) + + var artifactManifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+artifactDesc.Digest.Hex()].Data, &artifactManifest)) + require.Equal(t, artifactType, artifactManifest.ArtifactType) + require.Equal(t, "artifact manifest", artifactManifest.Annotations["am"]) + + var attestationManifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+attestationDesc.Digest.Hex()].Data, &attestationManifest)) + require.Equal(t, "application/vnd.docker.attestation.manifest.v1+json", attestationManifest.ArtifactType) + require.NotNil(t, attestationManifest.Subject) + require.Equal(t, artifactDesc.Digest, attestationManifest.Subject.Digest) + require.Len(t, attestationManifest.Layers, 1) + + var stmt provenanceStatement + require.NoError(t, json.Unmarshal(m[ocispecs.ImageBlobsDir+"/sha256/"+attestationManifest.Layers[0].Digest.Hex()].Data, &stmt)) + require.Len(t, stmt.Predicate.BuildDefinition.InternalParameters.Postprocess, 1) + require.Equal(t, "artifact.v0", stmt.Predicate.BuildDefinition.InternalParameters.Postprocess[0].Frontend) + }) + + t.Run("image", func(t *testing.T) { + workers.CheckFeatureCompat(t, sb, workers.FeatureDirectPush) + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + target := registry + "/buildkit/testartifactpostprocessor:latest" + resp, err := c.Build(sb.Context(), SolveOpt{ + Exports: []ExportEntry{{ + Type: ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }}, + }, "", artifactFrontend(configMediaType, layerMediaType), nil) + require.NoError(t, err) + + desc := decodeDescriptor(t, resp.ExporterResponse[exptypes.ExporterImageDescriptorKey]) + require.Equal(t, artifactType, desc.ArtifactType) + require.Equal(t, desc.Digest.String(), resp.ExporterResponse[exptypes.ExporterImageDigestKey]) + require.Equal(t, expectedConfigDigest.String(), resp.ExporterResponse[exptypes.ExporterImageConfigDigestKey]) + + _, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + manifestData, err := content.ReadBlob(sb.Context(), provider, desc) + require.NoError(t, err) + + var manifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(manifestData, &manifest)) + require.Equal(t, artifactType, manifest.ArtifactType) + require.Equal(t, configMediaType, manifest.Config.MediaType) + require.Equal(t, expectedConfigDigest, manifest.Config.Digest) + require.Len(t, manifest.Layers, 1) + require.Equal(t, layerMediaType, manifest.Layers[0].MediaType) + }) + + t.Run("image-provenance", func(t *testing.T) { + workers.CheckFeatureCompat(t, sb, workers.FeatureDirectPush) + + registry, err := sb.NewRegistry() + if errors.Is(err, integration.ErrRequirements) { + t.Skip(err.Error()) + } + require.NoError(t, err) + + target := registry + "/buildkit/testartifactpostprocessor-provenance:latest" + resp, err := c.Build(sb.Context(), SolveOpt{ + FrontendAttrs: map[string]string{ + "attest:provenance": "mode=max,version=v1", + }, + Exports: []ExportEntry{{ + Type: ExporterImage, + Attrs: map[string]string{ + "name": target, + "push": "true", + }, + }}, + }, "", artifactFrontend(configMediaType, layerMediaType), nil) + require.NoError(t, err) + + desc := decodeDescriptor(t, resp.ExporterResponse[exptypes.ExporterImageDescriptorKey]) + require.Equal(t, ocispecs.MediaTypeImageIndex, desc.MediaType) + require.Equal(t, desc.Digest.String(), resp.ExporterResponse[exptypes.ExporterImageDigestKey]) + require.Equal(t, expectedConfigDigest.String(), resp.ExporterResponse[exptypes.ExporterImageConfigDigestKey]) + + _, provider, err := contentutil.ProviderFromRef(target) + require.NoError(t, err) + + indexData, err := content.ReadBlob(sb.Context(), provider, desc) + require.NoError(t, err) + + var index ocispecs.Index + require.NoError(t, json.Unmarshal(indexData, &index)) + require.Len(t, index.Manifests, 2) + + artifactDesc, attestationDesc := findArtifactDescriptors(t, index.Manifests) + require.Equal(t, artifactType, artifactDesc.ArtifactType) + require.Equal(t, "unknown/unknown", platforms.Format(*attestationDesc.Platform)) + require.Equal(t, attestation.DockerAnnotationReferenceTypeDefault, attestationDesc.Annotations[attestation.DockerAnnotationReferenceType]) + require.Equal(t, artifactDesc.Digest.String(), attestationDesc.Annotations[attestation.DockerAnnotationReferenceDigest]) + + artifactManifestData, err := content.ReadBlob(sb.Context(), provider, artifactDesc) + require.NoError(t, err) + var artifactManifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(artifactManifestData, &artifactManifest)) + require.Equal(t, artifactType, artifactManifest.ArtifactType) + require.Equal(t, configMediaType, artifactManifest.Config.MediaType) + require.Equal(t, expectedConfigDigest, artifactManifest.Config.Digest) + + attestationManifestData, err := content.ReadBlob(sb.Context(), provider, attestationDesc) + require.NoError(t, err) + var attestationManifest ocispecs.Manifest + require.NoError(t, json.Unmarshal(attestationManifestData, &attestationManifest)) + require.Equal(t, "application/vnd.docker.attestation.manifest.v1+json", attestationManifest.ArtifactType) + require.NotNil(t, attestationManifest.Subject) + require.Equal(t, artifactDesc.Digest, attestationManifest.Subject.Digest) + require.Len(t, attestationManifest.Layers, 1) + + stmtData, err := content.ReadBlob(sb.Context(), provider, attestationManifest.Layers[0]) + require.NoError(t, err) + var stmt provenanceStatement + require.NoError(t, json.Unmarshal(stmtData, &stmt)) + require.Len(t, stmt.Predicate.BuildDefinition.InternalParameters.Postprocess, 1) + require.Equal(t, "artifact.v0", stmt.Predicate.BuildDefinition.InternalParameters.Postprocess[0].Frontend) + }) + + t.Run("validation", func(t *testing.T) { + for _, tc := range []struct { + name string + configType string + layerType string + errorContains string + }{ + { + name: "missing-config-media-type", + layerType: layerMediaType, + errorContains: "config descriptor mediaType is required", + }, + { + name: "missing-layer-media-type", + configType: configMediaType, + errorContains: "layer 0 descriptor mediaType is required", + }, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + _, err := c.Build(sb.Context(), SolveOpt{ + Exports: []ExportEntry{{ + Type: ExporterLocal, + OutputDir: dir, + }}, + }, "", artifactFrontend(tc.configType, tc.layerType), nil) + require.Error(t, err) + require.Contains(t, err.Error(), tc.errorContains) + }) + } + }) +} + func testExportAttestationsOCIArtifact(t *testing.T, sb integration.Sandbox) { testExportAttestations(t, sb, true) } diff --git a/cmd/buildkitd/main.go b/cmd/buildkitd/main.go index 7de17c050559..9ed0efd39ee8 100644 --- a/cmd/buildkitd/main.go +++ b/cmd/buildkitd/main.go @@ -34,6 +34,7 @@ import ( "github.com/moby/buildkit/control" "github.com/moby/buildkit/executor/oci" "github.com/moby/buildkit/frontend" + artifactfrontend "github.com/moby/buildkit/frontend/artifact" dockerfile "github.com/moby/buildkit/frontend/dockerfile/builder" "github.com/moby/buildkit/frontend/gateway" "github.com/moby/buildkit/frontend/gateway/forwarder" @@ -817,6 +818,7 @@ func newController(ctx context.Context, c *cli.Context, cfg *config.Config) (*co if cfg.Frontends.Dockerfile.Enabled == nil || *cfg.Frontends.Dockerfile.Enabled { frontends["dockerfile.v0"] = forwarder.NewGatewayForwarder(wc.Infos(), dockerfile.Build) } + frontends[artifactfrontend.Name] = forwarder.NewGatewayForwarder(wc.Infos(), artifactfrontend.Build) if cfg.Frontends.Gateway.Enabled == nil || *cfg.Frontends.Gateway.Enabled { gwfe, err := gateway.NewGatewayFrontend(wc.Infos(), cfg.Frontends.Gateway.AllowedRepositories) if err != nil { diff --git a/control/control.go b/control/control.go index e91f566c7aff..c62d2310f9a0 100644 --- a/control/control.go +++ b/control/control.go @@ -516,6 +516,8 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (* procs = append(procs, proc.SBOMProcessor(ref.String(), useCache, resolveMode, params)) } + procs = append(procs, proc.ArtifactProcessor()) + if attrs, ok := attests["provenance"]; ok { var slsaVersion provenancetypes.ProvenanceSLSA params := make(map[string]string) diff --git a/exporter/containerimage/annotations.go b/exporter/containerimage/annotations.go index 1bf10f6c15db..a0e1e9906402 100644 --- a/exporter/containerimage/annotations.go +++ b/exporter/containerimage/annotations.go @@ -88,6 +88,27 @@ func (ag AnnotationsGroup) Platform(p *ocispecs.Platform) *Annotations { return res } +// ResolveArtifactAnnotations validates and resolves annotations for OCI +// artifact exports. Artifact exports always produce a single manifest +// descriptor, unless attestations are attached, in which case the top-level +// descriptor becomes an index that wraps the artifact manifest and attestation +// manifests. +func ResolveArtifactAnnotations(ag AnnotationsGroup, hasIndex bool) (*Annotations, error) { + resolved := ag.Platform(nil) + if !hasIndex && (len(resolved.Index) > 0 || len(resolved.IndexDescriptor) > 0) { + return nil, errors.Errorf("index annotations are not supported for OCI artifact exports without attestations") + } + for platform, annotations := range ag { + if platform == "" { + continue + } + if len(annotations.Manifest) > 0 || len(annotations.ManifestDescriptor) > 0 { + return nil, errors.Errorf("platform-specific annotations are not supported for OCI artifact exports") + } + } + return resolved, nil +} + func (ag AnnotationsGroup) Merge(other AnnotationsGroup) AnnotationsGroup { if other == nil { return ag diff --git a/exporter/containerimage/artifact.go b/exporter/containerimage/artifact.go new file mode 100644 index 000000000000..e2dcc125e64f --- /dev/null +++ b/exporter/containerimage/artifact.go @@ -0,0 +1,87 @@ +package containerimage + +import ( + "strconv" + "strings" + + intoto "github.com/in-toto/in-toto-golang/in_toto" + "github.com/moby/buildkit/exporter" + "github.com/moby/buildkit/exporter/attestation" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/solver/result" + "github.com/moby/buildkit/util/purl" + digest "github.com/opencontainers/go-digest" + "github.com/package-url/packageurl-go" + "github.com/pkg/errors" +) + +// ResolveArtifactAttestations selects the single attestation group that applies +// to an OCI artifact export. If forceInline is false, only inline-only +// attestations are retained, matching the regular image exporter behavior. +func ResolveArtifactAttestations(src *exporter.Source, forceInline bool) ([]exporter.Attestation, error) { + if len(src.Attestations) == 0 { + return nil, nil + } + + attestationGroups := make(map[string][]exporter.Attestation, len(src.Attestations)) + for k, atts := range src.Attestations { + if !forceInline { + atts = attestation.Filter(atts, nil, map[string][]byte{ + result.AttestationInlineOnlyKey: []byte(strconv.FormatBool(true)), + }) + } + if len(atts) > 0 { + attestationGroups[k] = atts + } + } + if len(attestationGroups) == 0 { + return nil, nil + } + if len(attestationGroups) == 1 { + for _, atts := range attestationGroups { + return atts, nil + } + } + + ps, err := exptypes.ParsePlatforms(src.Metadata) + if err != nil { + return nil, err + } + if len(ps.Platforms) != 1 { + return nil, errors.Errorf("OCI artifact exports require exactly one attestation group, got %d", len(attestationGroups)) + } + + atts, ok := attestationGroups[ps.Platforms[0].ID] + if !ok { + return nil, errors.Errorf("OCI artifact export missing attestation group for %s", ps.Platforms[0].ID) + } + if len(attestationGroups) != 1 { + return nil, errors.Errorf("OCI artifact exports require exactly one attestation group, got %d", len(attestationGroups)) + } + return atts, nil +} + +// DefaultArtifactSubjects creates default in-toto subjects for registry-pushed +// OCI artifacts. Artifact exports are platform-less, so no platform qualifier +// is added to the generated purls. +func DefaultArtifactSubjects(imageNames string, dgst digest.Digest) ([]intoto.Subject, error) { + if imageNames == "" { + return nil, nil + } + + var subjects []intoto.Subject + for name := range strings.SplitSeq(imageNames, ",") { + if name == "" { + continue + } + pl, err := purl.RefToPURL(packageurl.TypeDocker, name, nil) + if err != nil { + return nil, err + } + subjects = append(subjects, intoto.Subject{ + Name: pl, + Digest: result.ToDigestMap(dgst), + }) + } + return subjects, nil +} diff --git a/exporter/containerimage/export.go b/exporter/containerimage/export.go index 918b55e7cf29..63dd52b6e643 100644 --- a/exporter/containerimage/export.go +++ b/exporter/containerimage/export.go @@ -233,6 +233,12 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source } opts.Annotations = opts.Annotations.Merge(as) + // If the result contains a pre-built OCI layout (from ArtifactProcessor), + // use the dedicated OCI layout export path instead of the normal image commit. + if _, ok := src.Metadata[exptypes.ExporterOCILayoutKey]; ok { + return e.exportOCILayout(ctx, src, buildInfo, opts.Annotations) + } + ctx, done, err := leaseutil.WithLease(ctx, e.opt.LeaseManager, leaseutil.MakeTemporary) if err != nil { return nil, nil, nil, err @@ -435,6 +441,182 @@ func (e *imageExporterInstance) pushImage(ctx context.Context, src *exporter.Sou return push.Push(ctx, e.opt.SessionManager, sessionID, mprovider, e.opt.ImageWriter.ContentStore(), dgst, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest, annotations) } +// exportOCILayout handles exporting a pre-built OCI layout that was assembled +// by the ArtifactProcessor. It ingests the layout into the content store, +// stores the image if configured, and optionally pushes it. +func (e *imageExporterInstance) exportOCILayout(ctx context.Context, src *exporter.Source, buildInfo exporter.ExportBuildInfo, annotations AnnotationsGroup) (_ map[string]string, _ exporter.FinalizeFunc, descref exporter.DescriptorReference, err error) { + if e.unpack { + return nil, nil, nil, errors.New("unpack is not supported for OCI artifact layouts") + } + if e.opts.RewriteTimestamp { + return nil, nil, nil, errors.New("rewrite-timestamp is not supported for OCI artifact layouts") + } + + // Resolve the single ref from the source + var ref cache.ImmutableRef + if src.Ref != nil { + ref = src.Ref + } else if len(src.Refs) == 1 { + for _, r := range src.Refs { + ref = r + } + } + if ref == nil { + return nil, nil, nil, errors.New("OCI layout export requires exactly one ref") + } + + // Acquire a lease to protect content from GC during export + ctx, done, err := leaseutil.WithLease(ctx, e.opt.LeaseManager, leaseutil.MakeTemporary) + if err != nil { + return nil, nil, nil, err + } + defer func() { + if descref == nil { + done(context.WithoutCancel(ctx)) + } + }() + + if n, ok := src.Metadata["image.name"]; e.opts.ImageName == "*" && ok { + e.opts.ImageName = string(n) + } + + attests, err := ResolveArtifactAttestations(src, e.opts.ForceInlineAttestations) + if err != nil { + return nil, nil, nil, err + } + + resolvedAnnotations, err := ResolveArtifactAnnotations(annotations, len(attests) > 0) + if err != nil { + return nil, nil, nil, err + } + + desc, err := e.opt.ImageWriter.CommitOCILayout(ctx, ref, buildInfo.SessionID, resolvedAnnotations) + if err != nil { + return nil, nil, nil, err + } + if len(attests) > 0 { + subjects, err := DefaultArtifactSubjects(e.opts.ImageName, desc.Digest) + if err != nil { + return nil, nil, nil, err + } + desc, err = e.opt.ImageWriter.CommitAttestationsForDescriptor(ctx, &e.opts, *desc, buildInfo.SessionID, attests, subjects, resolvedAnnotations) + if err != nil { + return nil, nil, nil, err + } + } + + resp := make(map[string]string) + + nameCanonical := e.nameCanonical + if e.danglingPrefix != "" && (!e.danglingEmptyOnly || e.opts.ImageName == "") { + danglingImageName := e.danglingPrefix + "@" + desc.Digest.String() + if e.opts.ImageName != "" { + e.opts.ImageName += "," + danglingImageName + } else { + e.opts.ImageName = danglingImageName + nameCanonical = false + } + } + + // Collect names for finalize callback to push + var namesToPush []string + + if e.opts.ImageName != "" { + targetNames := strings.SplitSeq(e.opts.ImageName, ",") + for targetName := range targetNames { + if e.opt.Images != nil && e.store { + tagDone := progress.OneOff(ctx, "naming to "+targetName) + + // imageClientCtx is used for propagating the epoch to e.opt.Images.Update() and e.opt.Images.Create(). + // + // Ideally, we should be able to propagate the epoch via images.Image.CreatedAt. + // However, due to a bug of containerd, we are temporarily stuck with this workaround. + // https://github.com/containerd/containerd/issues/8322 + imageClientCtx := ctx + if e.opts.Epoch != nil { + imageClientCtx = epoch.WithSourceDateEpoch(imageClientCtx, e.opts.Epoch) + } + img := images.Image{ + Target: *desc, + // CreatedAt in images.Images is ignored due to a bug of containerd. + // See the comment lines for imageClientCtx. + } + + sfx := []string{""} + if nameCanonical && !strings.ContainsRune(targetName, '@') { + sfx = append(sfx, "@"+desc.Digest.String()) + } + for _, sfx := range sfx { + img.Name = targetName + sfx + for { // handle possible race between Update and Create + if _, err := e.opt.Images.Update(imageClientCtx, img); err != nil { + if !errors.Is(err, cerrdefs.ErrNotFound) { + return nil, nil, nil, tagDone(err) + } + + if _, err := e.opt.Images.Create(imageClientCtx, img); err != nil { + if !errors.Is(err, cerrdefs.ErrAlreadyExists) { + return nil, nil, nil, tagDone(err) + } + continue + } + } + break + } + } + tagDone(nil) + } + if e.push { + namesToPush = append(namesToPush, targetName) + } + } + resp[exptypes.ExporterImageNameKey] = e.opts.ImageName + } + + resp[exptypes.ExporterImageDigestKey] = desc.Digest.String() + if v, ok := desc.Annotations[exptypes.ExporterConfigDigestKey]; ok { + resp[exptypes.ExporterImageConfigDigestKey] = v + delete(desc.Annotations, exptypes.ExporterConfigDigestKey) + } + + dtdesc, err := json.Marshal(desc) + if err != nil { + return nil, nil, nil, err + } + resp[exptypes.ExporterImageDescriptorKey] = base64.StdEncoding.EncodeToString(dtdesc) + + // Transfer lease ownership to descref + descref = NewDescriptorReference(*desc, done) + + if len(namesToPush) == 0 { + return resp, nil, descref, nil + } + + // Create finalize callback for pushing + finalize := func(ctx context.Context) error { + for _, targetName := range namesToPush { + if err := e.pushOCILayout(ctx, buildInfo.SessionID, targetName, desc.Digest); err != nil { + var statusErr remoteserrors.ErrUnexpectedStatus + if errors.As(err, &statusErr) { + err = errutil.WithDetails(err) + } + return errors.Wrapf(err, "failed to push OCI layout to %v", targetName) + } + } + return nil + } + + return resp, finalize, descref, nil +} + +// pushOCILayout pushes an OCI artifact layout to a registry. +// Unlike pushImage, it does not need to resolve remote providers from cache refs +// because all blobs have already been ingested into the content store by CommitOCILayout. +func (e *imageExporterInstance) pushOCILayout(ctx context.Context, sessionID string, targetName string, dgst digest.Digest) error { + cs := e.opt.ImageWriter.ContentStore() + return push.Push(ctx, e.opt.SessionManager, sessionID, cs, cs, dgst, targetName, e.insecure, e.opt.RegistryHosts, e.pushByDigest, nil) +} + func (e *imageExporterInstance) unpackImage(ctx context.Context, img images.Image, src *exporter.Source, s session.Group) (err0 error) { matcher := platforms.Only(platforms.Normalize(platforms.DefaultSpec())) diff --git a/exporter/containerimage/exptypes/artifact.go b/exporter/containerimage/exptypes/artifact.go new file mode 100644 index 000000000000..d415e11d2442 --- /dev/null +++ b/exporter/containerimage/exptypes/artifact.go @@ -0,0 +1,11 @@ +package exptypes + +// ArtifactLayer describes a single layer in an OCI artifact. +// The frontend produces an array of these as JSON metadata, which the +// ArtifactProcessor passes to the OCI layout assembler container. +type ArtifactLayer struct { + // Path is the root-relative file path inside the artifact input reference. + Path string `json:"path"` + MediaType string `json:"mediaType"` + Annotations map[string]string `json:"annotations,omitempty"` +} diff --git a/exporter/containerimage/exptypes/types.go b/exporter/containerimage/exptypes/types.go index d7cc98b9485e..8ffc4be740f5 100644 --- a/exporter/containerimage/exptypes/types.go +++ b/exporter/containerimage/exptypes/types.go @@ -16,6 +16,15 @@ const ( ExporterImageDescriptorKey = "containerimage.descriptor" ExporterImageBaseConfigKey = "containerimage.base.config" ExporterPlatformsKey = "refs.platforms" + + ExporterArtifactKey = "containerimage.artifact" + ExporterArtifactTypeKey = "containerimage.artifact.type" + ExporterArtifactConfigTypeKey = "containerimage.artifact.config.mediatype" + ExporterArtifactLayersKey = "containerimage.artifact.layers" + ExporterPostprocessorsKey = "containerimage.postprocessors" + ExporterOCILayoutKey = "containerimage.oci-layout" + + PostprocessInputKey = "input" ) // KnownRefMetadataKeys are the subset of exporter keys that can be suffixed by @@ -38,3 +47,8 @@ type InlineCacheEntry struct { Data []byte } type InlineCache func(ctx context.Context) (*result.Result[*InlineCacheEntry], error) + +type PostprocessRequest struct { + Frontend string `json:"frontend"` + FrontendOpt map[string]string `json:"frontendOpt,omitempty"` +} diff --git a/exporter/containerimage/writer.go b/exporter/containerimage/writer.go index f0cddeab91be..5dbec06d65a2 100644 --- a/exporter/containerimage/writer.go +++ b/exporter/containerimage/writer.go @@ -5,6 +5,8 @@ import ( "context" "encoding/json" "fmt" + "maps" + "os" "reflect" "strconv" "strings" @@ -14,6 +16,7 @@ import ( "github.com/containerd/containerd/v2/core/diff" "github.com/containerd/containerd/v2/core/images" "github.com/containerd/containerd/v2/pkg/labels" + "github.com/containerd/continuity/fs" "github.com/containerd/platforms" intoto "github.com/in-toto/in-toto-golang/in_toto" "github.com/moby/buildkit/cache" @@ -660,6 +663,87 @@ func (ic *ImageWriter) commitAttestationsManifest(ctx context.Context, opts *Ima }, nil } +// CommitAttestationsForDescriptor wraps an existing artifact/image manifest +// descriptor in an OCI index alongside its attestation manifest. +func (ic *ImageWriter) CommitAttestationsForDescriptor(ctx context.Context, opts *ImageCommitOpts, target ocispecs.Descriptor, sessionID string, attestations []exporter.Attestation, defaultSubjects []intoto.Subject, annotations *Annotations) (*ocispecs.Descriptor, error) { + if len(attestations) == 0 { + return &target, nil + } + if annotations == nil { + annotations = &Annotations{} + } + + opts.EnableOCITypes(ctx, "attestations") + + attestations, err := attestation.Unbundle(ctx, session.NewGroup(sessionID), attestations) + if err != nil { + return nil, err + } + + stmts, err := attestation.MakeInTotoStatements(ctx, session.NewGroup(sessionID), attestations, defaultSubjects) + if err != nil { + return nil, err + } + + attestationDesc, err := ic.commitAttestationsManifest(ctx, opts, target, stmts, true) + if err != nil { + return nil, err + } + attestationDesc.Platform = &intotoPlatform + + indexTarget := target + if len(target.Annotations) > 0 { + indexTarget.Annotations = maps.Clone(target.Annotations) + delete(indexTarget.Annotations, exptypes.ExporterConfigDigestKey) + if len(indexTarget.Annotations) == 0 { + indexTarget.Annotations = nil + } + } + + idx := ocispecs.Index{ + MediaType: ocispecs.MediaTypeImageIndex, + Annotations: maps.Clone(annotations.Index), + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + Manifests: []ocispecs.Descriptor{ + indexTarget, + *attestationDesc, + }, + } + idxLabels := map[string]string{ + "containerd.io/gc.ref.content.0": target.Digest.String(), + "containerd.io/gc.ref.content.1": attestationDesc.Digest.String(), + } + + idxBytes, err := json.MarshalIndent(idx, "", " ") + if err != nil { + return nil, errors.Wrap(err, "failed to marshal artifact attestation index") + } + + idxDigest := digest.FromBytes(idxBytes) + idxDesc := ocispecs.Descriptor{ + Digest: idxDigest, + Size: int64(len(idxBytes)), + MediaType: idx.MediaType, + Annotations: maps.Clone(annotations.IndexDescriptor), + } + if v, ok := target.Annotations[exptypes.ExporterConfigDigestKey]; ok { + if idxDesc.Annotations == nil { + idxDesc.Annotations = map[string]string{} + } + idxDesc.Annotations[exptypes.ExporterConfigDigestKey] = v + } + + done := progress.OneOff(ctx, "exporting attestation index "+idxDigest.String()) + if err := content.WriteBlob(ctx, ic.opt.ContentStore, idxDigest.String(), bytes.NewReader(idxBytes), idxDesc, content.WithLabels(idxLabels)); err != nil { + return nil, done(errors.Wrapf(err, "error writing attestation index blob %s", idxDigest)) + } + done(nil) + + return &idxDesc, nil +} + func (ic *ImageWriter) ContentStore() content.Store { return ic.opt.ContentStore } @@ -950,3 +1034,185 @@ func getRefMetadata(ref cache.ImmutableRef, limit int) []refMetadata { } return metas } + +// CommitOCILayout reads an OCI image layout from the given ref's filesystem, +// ingests all blobs (config, layers, manifest) into the content store, and +// returns the manifest descriptor. If annotations is non-nil, manifest and +// manifest-descriptor annotations are applied before ingestion. +func (ic *ImageWriter) CommitOCILayout(ctx context.Context, ref cache.ImmutableRef, sessionID string, annotations *Annotations) (*ocispecs.Descriptor, error) { + if ref == nil { + return nil, errors.New("cannot commit OCI layout: ref is nil") + } + + // Mount the ref to get filesystem access + mount, err := ref.Mount(ctx, true, session.NewGroup(sessionID)) + if err != nil { + return nil, errors.Wrap(err, "failed to mount ref for OCI layout") + } + lm := snapshot.LocalMounter(mount) + rootDir, err := lm.Mount() + if err != nil { + return nil, errors.Wrap(err, "failed to locally mount ref for OCI layout") + } + defer lm.Unmount() + + // Helper to read a small file safely from the mounted root + readFileFromRoot := func(p string) ([]byte, error) { + fp, err := fs.RootPath(rootDir, p) + if err != nil { + return nil, errors.Wrapf(err, "failed to resolve path %s", p) + } + return os.ReadFile(fp) + } + + // Helper to open a file safely from the mounted root for streaming + openFileFromRoot := func(p string) (*os.File, error) { + fp, err := fs.RootPath(rootDir, p) + if err != nil { + return nil, errors.Wrapf(err, "failed to resolve path %s", p) + } + return os.Open(fp) + } + blobPath := func(d digest.Digest) string { + return ocispecs.ImageBlobsDir + "/" + d.Algorithm().String() + "/" + d.Encoded() + } + + cs := ic.opt.ContentStore + + // 1. Read and validate oci-layout file + layoutData, err := readFileFromRoot(ocispecs.ImageLayoutFile) + if err != nil { + return nil, errors.Wrap(err, "failed to read oci-layout file") + } + var layout ocispecs.ImageLayout + if err := json.Unmarshal(layoutData, &layout); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal oci-layout") + } + if layout.Version != ocispecs.ImageLayoutVersion { + return nil, errors.Errorf("unsupported OCI image layout version: %s", layout.Version) + } + + // 2. Read index.json + indexData, err := readFileFromRoot(ocispecs.ImageIndexFile) + if err != nil { + return nil, errors.Wrap(err, "failed to read index.json") + } + var idx ocispecs.Index + if err := json.Unmarshal(indexData, &idx); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal index.json") + } + + // 3. Validate: must have exactly 1 manifest + if len(idx.Manifests) != 1 { + return nil, errors.Errorf("OCI layout index.json must contain exactly 1 manifest, got %d", len(idx.Manifests)) + } + mfstDesc := idx.Manifests[0] + if mfstDesc.MediaType == "" { + return nil, errors.New("OCI layout manifest descriptor mediaType is required") + } + + // 4. Read the manifest blob + mfstData, err := readFileFromRoot(blobPath(mfstDesc.Digest)) + if err != nil { + return nil, errors.Wrapf(err, "failed to read manifest blob %s", mfstDesc.Digest) + } + var mfst ocispecs.Manifest + if err := json.Unmarshal(mfstData, &mfst); err != nil { + return nil, errors.Wrap(err, "failed to unmarshal manifest") + } + if mfst.Config.MediaType == "" { + return nil, errors.New("OCI layout config descriptor mediaType is required") + } + for i, layer := range mfst.Layers { + if layer.MediaType == "" { + return nil, errors.Errorf("OCI layout layer %d descriptor mediaType is required", i) + } + } + switch { + case mfstDesc.ArtifactType == "": + mfstDesc.ArtifactType = mfst.ArtifactType + case mfst.ArtifactType != "" && mfstDesc.ArtifactType != mfst.ArtifactType: + return nil, errors.New("OCI layout manifest descriptor artifactType does not match manifest artifactType") + } + + // Apply user-requested manifest annotations. This changes the manifest + // content, so we must re-serialize and recompute the digest/size. + if annotations != nil && len(annotations.Manifest) > 0 { + if mfst.Annotations == nil { + mfst.Annotations = make(map[string]string) + } + for k, v := range annotations.Manifest { + mfst.Annotations[k] = v + } + mfstData, err = json.Marshal(mfst) + if err != nil { + return nil, errors.Wrap(err, "failed to re-marshal manifest with annotations") + } + mfstDesc.Digest = digest.FromBytes(mfstData) + mfstDesc.Size = int64(len(mfstData)) + } + + if annotations != nil && len(annotations.ManifestDescriptor) > 0 { + if mfstDesc.Annotations == nil { + mfstDesc.Annotations = make(map[string]string) + } + for k, v := range annotations.ManifestDescriptor { + mfstDesc.Annotations[k] = v + } + } + + // 5. Ingest config blob + configData, err := readFileFromRoot(blobPath(mfst.Config.Digest)) + if err != nil { + return nil, errors.Wrapf(err, "failed to read config blob %s", mfst.Config.Digest) + } + configDone := progress.OneOff(ctx, "ingesting OCI layout config "+mfst.Config.Digest.String()) + if err := content.WriteBlob(ctx, cs, mfst.Config.Digest.String(), bytes.NewReader(configData), mfst.Config); err != nil { + return nil, configDone(errors.Wrapf(err, "failed to ingest config blob %s", mfst.Config.Digest)) + } + configDone(nil) + + // 6. Ingest layer blobs — stream from mounted files to avoid buffering + // large artifacts (e.g. multi-GB model files) entirely in memory. + for i, layer := range mfst.Layers { + layerFile, err := openFileFromRoot(blobPath(layer.Digest)) + if err != nil { + return nil, errors.Wrapf(err, "failed to open layer blob %s", layer.Digest) + } + layerDone := progress.OneOff(ctx, fmt.Sprintf("ingesting OCI layout layer %d/%d %s", i+1, len(mfst.Layers), layer.Digest.String())) + err = content.WriteBlob(ctx, cs, layer.Digest.String(), layerFile, layer) + layerFile.Close() + if err != nil { + return nil, layerDone(errors.Wrapf(err, "failed to ingest layer blob %s", layer.Digest)) + } + layerDone(nil) + } + + // 7. Ingest manifest blob with GC reference labels + gcLabels := map[string]string{ + "containerd.io/gc.ref.content.config": mfst.Config.Digest.String(), + } + for i, l := range mfst.Layers { + gcLabels[fmt.Sprintf("containerd.io/gc.ref.content.l.%d", i)] = l.Digest.String() + } + + mfstBlobDesc := ocispecs.Descriptor{ + Digest: mfstDesc.Digest, + Size: int64(len(mfstData)), + MediaType: mfstDesc.MediaType, + } + mfstDone := progress.OneOff(ctx, "ingesting OCI layout manifest "+mfstDesc.Digest.String()) + if err := content.WriteBlob(ctx, cs, mfstDesc.Digest.String(), bytes.NewReader(mfstData), mfstBlobDesc, content.WithLabels(gcLabels)); err != nil { + return nil, mfstDone(errors.Wrapf(err, "failed to ingest manifest blob %s", mfstDesc.Digest)) + } + mfstDone(nil) + + if mfstDesc.Annotations == nil { + mfstDesc.Annotations = make(map[string]string) + } + mfstDesc.Annotations[exptypes.ExporterConfigDigestKey] = mfst.Config.Digest.String() + + bklog.G(ctx).Debugf("committed OCI layout: manifest=%s config=%s layers=%d", mfstDesc.Digest, mfst.Config.Digest, len(mfst.Layers)) + + return &mfstDesc, nil +} diff --git a/exporter/oci/export.go b/exporter/oci/export.go index 24b96a338a1f..c30fc020a197 100644 --- a/exporter/oci/export.go +++ b/exporter/oci/export.go @@ -149,6 +149,21 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source } opts.Annotations = opts.Annotations.Merge(as) + if _, ok := src.Metadata[exptypes.ExporterOCILayoutKey]; ok { + if e.opt.Variant == VariantDocker { + return nil, nil, nil, errors.New("docker exporter does not support OCI artifact layouts") + } + attests, err := containerimage.ResolveArtifactAttestations(src, e.opts.ForceInlineAttestations) + if err != nil { + return nil, nil, nil, err + } + resolvedAnnotations, err := containerimage.ResolveArtifactAnnotations(opts.Annotations, len(attests) > 0) + if err != nil { + return nil, nil, nil, err + } + return e.exportOCILayout(ctx, src, buildInfo, resolvedAnnotations, attests) + } + ctx, done, err := leaseutil.WithLease(ctx, e.opt.LeaseManager, leaseutil.MakeTemporary) if err != nil { return nil, nil, nil, err @@ -295,6 +310,123 @@ func (e *imageExporterInstance) Export(ctx context.Context, src *exporter.Source return resp, nil, nil, nil } +func (e *imageExporterInstance) exportOCILayout(ctx context.Context, src *exporter.Source, buildInfo exporter.ExportBuildInfo, annotations *containerimage.Annotations, attestations []exporter.Attestation) (_ map[string]string, _ exporter.FinalizeFunc, _ exporter.DescriptorReference, err error) { + ctx, done, err := leaseutil.WithLease(ctx, e.opt.LeaseManager, leaseutil.MakeTemporary) + if err != nil { + return nil, nil, nil, err + } + defer done(context.WithoutCancel(ctx)) + + var ref cache.ImmutableRef + if src.Ref != nil { + ref = src.Ref + } else if len(src.Refs) == 1 { + for _, r := range src.Refs { + ref = r + } + } + if ref == nil { + return nil, nil, nil, errors.New("OCI layout export requires exactly one ref") + } + + desc, err := e.opt.ImageWriter.CommitOCILayout(ctx, ref, buildInfo.SessionID, annotations) + if err != nil { + return nil, nil, nil, err + } + if n, ok := src.Metadata["image.name"]; e.opts.ImageName == "*" && ok { + e.opts.ImageName = string(n) + } + if len(attestations) > 0 { + subjects, err := containerimage.DefaultArtifactSubjects(e.opts.ImageName, desc.Digest) + if err != nil { + return nil, nil, nil, err + } + desc, err = e.opt.ImageWriter.CommitAttestationsForDescriptor(ctx, &e.opts, *desc, buildInfo.SessionID, attestations, subjects, annotations) + if err != nil { + return nil, nil, nil, err + } + } + + if desc.Annotations == nil { + desc.Annotations = map[string]string{} + } + if _, ok := desc.Annotations[ocispecs.AnnotationCreated]; !ok { + tm := time.Now() + if e.opts.Epoch != nil { + tm = *e.opts.Epoch + } + desc.Annotations[ocispecs.AnnotationCreated] = tm.UTC().Format(time.RFC3339) + } + + resp := make(map[string]string) + resp[exptypes.ExporterImageDigestKey] = desc.Digest.String() + if v, ok := desc.Annotations[exptypes.ExporterConfigDigestKey]; ok { + resp[exptypes.ExporterImageConfigDigestKey] = v + delete(desc.Annotations, exptypes.ExporterConfigDigestKey) + } + + dtdesc, err := json.Marshal(desc) + if err != nil { + return nil, nil, nil, err + } + resp[exptypes.ExporterImageDescriptorKey] = base64.StdEncoding.EncodeToString(dtdesc) + + names, err := normalizedNames(e.opts.ImageName) + if err != nil { + return nil, nil, nil, err + } + if len(names) != 0 { + resp[exptypes.ExporterImageNameKey] = strings.Join(names, ",") + } + + expOpts := []archiveexporter.ExportOpt{ + archiveexporter.WithManifest(*desc, names...), + archiveexporter.WithAllPlatforms(), + archiveexporter.WithSkipDockerManifest(), + } + + timeoutCtx, cancel := context.WithCancelCause(ctx) + timeoutCtx, _ = context.WithTimeoutCause(timeoutCtx, 5*time.Second, errors.WithStack(context.DeadlineExceeded)) //nolint:govet + defer func() { cancel(errors.WithStack(context.Canceled)) }() + + caller, err := e.opt.SessionManager.Get(timeoutCtx, buildInfo.SessionID, false) + if err != nil { + return nil, nil, nil, err + } + + cs := e.opt.ImageWriter.ContentStore() + if e.tar { + w, err := filesync.CopyFileWriter(ctx, resp, e.id, caller) + if err != nil { + return nil, nil, nil, err + } + + report := progress.OneOff(ctx, "sending tarball") + if err := archiveexporter.Export(ctx, cs, w, expOpts...); err != nil { + w.Close() + if grpcerrors.Code(err) == codes.AlreadyExists { + return resp, nil, nil, report(nil) + } + return nil, nil, nil, report(err) + } + err = w.Close() + if grpcerrors.Code(err) == codes.AlreadyExists { + return resp, nil, nil, report(nil) + } + if err != nil { + return nil, nil, nil, report(err) + } + report(nil) + } else { + store := sessioncontent.NewCallerStore(caller, "export") + if err := contentutil.CopyChain(ctx, store, cs, *desc); err != nil { + return nil, nil, nil, err + } + } + + return resp, nil, nil, nil +} + func normalizedNames(name string) ([]string, error) { if name == "" { return nil, nil diff --git a/frontend/artifact/artifact.go b/frontend/artifact/artifact.go new file mode 100644 index 000000000000..f1f023ae8de6 --- /dev/null +++ b/frontend/artifact/artifact.go @@ -0,0 +1,239 @@ +package artifact + +import ( + "context" + "encoding/json" + "path" + + "github.com/moby/buildkit/client/llb" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + gateway "github.com/moby/buildkit/frontend/gateway/client" + digest "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go" + ocispecs "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +const ( + Name = "artifact.v0" + InputKey = "input" + + digestChunkSize = 4 << 20 +) + +func Build(ctx context.Context, c gateway.Client) (*gateway.Result, error) { + inputState, err := resolveInputState(ctx, c) + if err != nil { + return nil, err + } + + opts := c.BuildOpts().Opts + artifactType := opts[exptypes.ExporterArtifactTypeKey] + configMediaType := opts[exptypes.ExporterArtifactConfigTypeKey] + + var layers []exptypes.ArtifactLayer + if err := json.Unmarshal([]byte(opts[exptypes.ExporterArtifactLayersKey]), &layers); err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to parse layers metadata") + } + layers, err = normalizeAndValidateArtifactMetadata(configMediaType, layers) + if err != nil { + return nil, err + } + + inputDef, err := inputState.Marshal(ctx) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to marshal input state") + } + inputRes, err := c.Solve(ctx, gateway.SolveRequest{ + Definition: inputDef.ToPB(), + }) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to solve input state") + } + inputRef, err := inputRes.SingleRef() + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to resolve input ref") + } + + configContent := []byte("{}") + configDigest := digest.FromBytes(configContent) + configDesc := ocispecs.Descriptor{ + MediaType: configMediaType, + Digest: configDigest, + Size: int64(len(configContent)), + } + + st := llb.Scratch(). + File(llb.Mkdir("/blobs/sha256", 0755, llb.WithParents(true))). + File(llb.Mkfile("/blobs/sha256/"+configDigest.Encoded(), 0644, configContent)) + + layerDescs := make([]ocispecs.Descriptor, 0, len(layers)) + for _, layer := range layers { + stat, err := inputRef.StatFile(ctx, gateway.StatRequest{Path: layer.Path}) + if err != nil { + return nil, errors.Wrapf(err, "artifact frontend: failed to stat layer file %s relative to input root", layer.Path) + } + if stat.IsDir() { + return nil, errors.Errorf("artifact frontend: layer path %s must be a file", layer.Path) + } + + dgst, err := digestFile(ctx, inputRef, layer.Path, stat.Size) + if err != nil { + return nil, err + } + desc := ocispecs.Descriptor{ + MediaType: layer.MediaType, + Digest: dgst, + Size: stat.Size, + Annotations: layer.Annotations, + } + layerDescs = append(layerDescs, desc) + + st = st.File(llb.Copy(inputState, layer.Path, "/blobs/sha256/"+dgst.Encoded(), &llb.CopyInfo{ + FollowSymlinks: true, + })) + } + + manifest := ocispecs.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + MediaType: ocispecs.MediaTypeImageManifest, + Config: configDesc, + Layers: layerDescs, + } + if artifactType != "" { + manifest.ArtifactType = artifactType + } + + manifestJSON, err := json.Marshal(manifest) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to marshal manifest") + } + manifestDigest := digest.FromBytes(manifestJSON) + + index := ocispecs.Index{ + Versioned: specs.Versioned{SchemaVersion: 2}, + Manifests: []ocispecs.Descriptor{{ + MediaType: ocispecs.MediaTypeImageManifest, + Digest: manifestDigest, + Size: int64(len(manifestJSON)), + ArtifactType: artifactType, + }}, + } + indexJSON, err := json.Marshal(index) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to marshal index") + } + + layoutJSON, err := json.Marshal(ocispecs.ImageLayout{Version: ocispecs.ImageLayoutVersion}) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to marshal oci-layout") + } + + st = st.File(llb.Mkfile("/blobs/sha256/"+manifestDigest.Encoded(), 0644, manifestJSON)) + st = st.File(llb.Mkfile("/index.json", 0644, indexJSON)) + st = st.File(llb.Mkfile("/oci-layout", 0644, layoutJSON)) + + def, err := st.Marshal(ctx, llb.WithCustomName("[artifact] assembling OCI layout")) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to marshal OCI layout LLB") + } + + res, err := c.Solve(ctx, gateway.SolveRequest{ + Definition: def.ToPB(), + }) + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: failed to solve OCI layout assembly") + } + ref, err := res.SingleRef() + if err != nil { + return nil, errors.Wrap(err, "artifact frontend: solve returned no ref") + } + + out := gateway.NewResult() + out.SetRef(ref) + return out, nil +} + +func resolveInputState(ctx context.Context, c gateway.Client) (llb.State, error) { + inputs, err := c.Inputs(ctx) + if err != nil { + return llb.State{}, errors.Wrap(err, "artifact frontend: failed to load inputs") + } + if st, ok := inputs[InputKey]; ok { + return st, nil + } + if len(inputs) == 1 { + for _, st := range inputs { + return st, nil + } + } + return llb.State{}, errors.New("artifact frontend: missing input state") +} + +func normalizeAndValidateArtifactMetadata(configMediaType string, layers []exptypes.ArtifactLayer) ([]exptypes.ArtifactLayer, error) { + if configMediaType == "" { + return nil, errors.New("artifact frontend: config descriptor mediaType is required") + } + normalized := make([]exptypes.ArtifactLayer, len(layers)) + for i, layer := range layers { + cleanedPath, err := normalizeLayerPath(layer.Path) + if err != nil { + return nil, errors.Wrapf(err, "artifact frontend: invalid layer %d path", i) + } + if layer.MediaType == "" { + return nil, errors.Errorf("artifact frontend: layer %d descriptor mediaType is required", i) + } + layer.Path = cleanedPath + normalized[i] = layer + } + return normalized, nil +} + +func normalizeLayerPath(p string) (string, error) { + if p == "" { + return "", errors.New("path is required") + } + cleaned := path.Clean(p) + switch cleaned { + case ".", "/": + return "", errors.New("path must reference a file within the artifact input root") + case "..": + return "", errors.New("path must be relative to the artifact input root") + } + if cleaned[0] == '/' || len(cleaned) >= 3 && cleaned[:3] == "../" { + return "", errors.New("path must be relative to the artifact input root") + } + return cleaned, nil +} + +func digestFile(ctx context.Context, ref gateway.Reference, path string, size int64) (digest.Digest, error) { + digester := digest.Canonical.Digester() + hash := digester.Hash() + + for offset := int64(0); offset < size; { + length := digestChunkSize + if remaining := size - offset; remaining < int64(length) { + length = int(remaining) + } + + dt, err := ref.ReadFile(ctx, gateway.ReadRequest{ + Filename: path, + Range: &gateway.FileRange{ + Offset: int(offset), + Length: length, + }, + }) + if err != nil { + return "", errors.Wrapf(err, "artifact frontend: failed to read layer file %s", path) + } + if len(dt) == 0 && length > 0 { + return "", errors.Errorf("artifact frontend: short read while hashing %s", path) + } + if _, err := hash.Write(dt); err != nil { + return "", errors.Wrapf(err, "artifact frontend: failed to hash layer file %s", path) + } + offset += int64(len(dt)) + } + + return digester.Digest(), nil +} diff --git a/frontend/artifact/artifact_test.go b/frontend/artifact/artifact_test.go new file mode 100644 index 000000000000..1ec028018a85 --- /dev/null +++ b/frontend/artifact/artifact_test.go @@ -0,0 +1,63 @@ +package artifact + +import ( + "testing" + + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/stretchr/testify/require" +) + +func TestNormalizeAndValidateArtifactMetadata(t *testing.T) { + t.Run("normalizes relative paths", func(t *testing.T) { + layers, err := normalizeAndValidateArtifactMetadata("application/vnd.test.config.v1+json", []exptypes.ArtifactLayer{{ + Path: "./subdir/../artifact.txt", + MediaType: "application/vnd.test.layer.v1", + }}) + require.NoError(t, err) + require.Len(t, layers, 1) + require.Equal(t, "artifact.txt", layers[0].Path) + }) + + t.Run("rejects missing config media type", func(t *testing.T) { + _, err := normalizeAndValidateArtifactMetadata("", nil) + require.EqualError(t, err, "artifact frontend: config descriptor mediaType is required") + }) + + t.Run("rejects missing layer path", func(t *testing.T) { + _, err := normalizeAndValidateArtifactMetadata("application/vnd.test.config.v1+json", []exptypes.ArtifactLayer{{ + MediaType: "application/vnd.test.layer.v1", + }}) + require.EqualError(t, err, "artifact frontend: invalid layer 0 path: path is required") + }) + + t.Run("rejects absolute paths", func(t *testing.T) { + _, err := normalizeAndValidateArtifactMetadata("application/vnd.test.config.v1+json", []exptypes.ArtifactLayer{{ + Path: "/artifact.txt", + MediaType: "application/vnd.test.layer.v1", + }}) + require.EqualError(t, err, "artifact frontend: invalid layer 0 path: path must be relative to the artifact input root") + }) + + t.Run("rejects parent traversal", func(t *testing.T) { + _, err := normalizeAndValidateArtifactMetadata("application/vnd.test.config.v1+json", []exptypes.ArtifactLayer{{ + Path: "../artifact.txt", + MediaType: "application/vnd.test.layer.v1", + }}) + require.EqualError(t, err, "artifact frontend: invalid layer 0 path: path must be relative to the artifact input root") + }) + + t.Run("rejects root path", func(t *testing.T) { + _, err := normalizeAndValidateArtifactMetadata("application/vnd.test.config.v1+json", []exptypes.ArtifactLayer{{ + Path: ".", + MediaType: "application/vnd.test.layer.v1", + }}) + require.EqualError(t, err, "artifact frontend: invalid layer 0 path: path must reference a file within the artifact input root") + }) + + t.Run("rejects missing layer media type", func(t *testing.T) { + _, err := normalizeAndValidateArtifactMetadata("application/vnd.test.config.v1+json", []exptypes.ArtifactLayer{{ + Path: "artifact.txt", + }}) + require.EqualError(t, err, "artifact frontend: layer 0 descriptor mediaType is required") + }) +} diff --git a/solver/llbsolver/proc/artifact.go b/solver/llbsolver/proc/artifact.go new file mode 100644 index 000000000000..f490a96b1d03 --- /dev/null +++ b/solver/llbsolver/proc/artifact.go @@ -0,0 +1,149 @@ +package proc + +import ( + "context" + "encoding/json" + "maps" + + "github.com/moby/buildkit/executor/resources" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/frontend" + artifactfrontend "github.com/moby/buildkit/frontend/artifact" + "github.com/moby/buildkit/solver" + "github.com/moby/buildkit/solver/llbsolver" + solverpb "github.com/moby/buildkit/solver/pb" + "github.com/moby/buildkit/util/tracing" + "github.com/pkg/errors" +) + +func ArtifactProcessor() llbsolver.Processor { + return func(ctx context.Context, res *llbsolver.Result, s *llbsolver.Solver, j *solver.Job, usage *resources.SysSampler) (_ *llbsolver.Result, err error) { + if _, ok := res.Metadata[exptypes.ExporterArtifactKey]; !ok { + return res, nil + } + + span, ctx := tracing.StartSpan(ctx, "create artifact OCI layout") + defer span.End() + + if len(res.Refs) > 1 { + return nil, errors.New("artifact processor requires exactly one ref") + } + + ref, ok := res.FindRef("") + if !ok || ref == nil { + return nil, errors.New("artifact processor requires exactly one ref") + } + + postRes, err := s.SolvePostprocessor(ctx, j, frontend.SolveRequest{ + Frontend: artifactfrontend.Name, + FrontendOpt: map[string]string{ + exptypes.ExporterArtifactTypeKey: string(res.Metadata[exptypes.ExporterArtifactTypeKey]), + exptypes.ExporterArtifactConfigTypeKey: string(res.Metadata[exptypes.ExporterArtifactConfigTypeKey]), + exptypes.ExporterArtifactLayersKey: string(res.Metadata[exptypes.ExporterArtifactLayersKey]), + }, + FrontendInputs: map[string]*solverpb.Definition{ + artifactfrontend.InputKey: ref.Definition(), + }, + }) + if err != nil { + return nil, errors.Wrap(err, "artifact processor: failed to run artifact frontend") + } + + if err := updateArtifactProvenance(res, postRes); err != nil { + return nil, err + } + postMeta := maps.Clone(res.Metadata) + if err := normalizeArtifactResultMetadata(postMeta); err != nil { + return nil, err + } + postMeta[exptypes.ExporterOCILayoutKey] = []byte("true") + delete(postMeta, exptypes.ExporterArtifactKey) + delete(postMeta, exptypes.ExporterArtifactTypeKey) + delete(postMeta, exptypes.ExporterArtifactConfigTypeKey) + delete(postMeta, exptypes.ExporterArtifactLayersKey) + if postRes.Result.Metadata == nil { + postRes.Result.Metadata = map[string][]byte{} + } + maps.Copy(postRes.Result.Metadata, postMeta) + if len(res.Attestations) > 0 { + if postRes.Result.Attestations == nil { + postRes.Result.Attestations = map[string][]frontend.Attestation{} + } + maps.Copy(postRes.Result.Attestations, res.Attestations) + } + + res.Result = postRes.Result + + if err := ref.Release(context.WithoutCancel(ctx)); err != nil { + return nil, errors.Wrap(err, "artifact processor: failed to release original result ref") + } + + return res, nil + } +} + +func updateArtifactProvenance(res *llbsolver.Result, postRes *llbsolver.Result) error { + if postRes == nil || postRes.Provenance == nil { + if res.Provenance != nil { + res.Provenance.Refs = nil + } + return nil + } + if res.Provenance == nil { + res.Provenance = postRes.Provenance + res.Provenance.Refs = nil + return nil + } + + oldProv := res.Provenance.Ref + newProv := postRes.Provenance.Ref + if newProv == nil { + res.Provenance.Ref = oldProv + res.Provenance.Refs = nil + return nil + } + if oldProv != nil { + postprocess := newProv.Parameters() + if err := newProv.Merge(oldProv); err != nil { + return errors.Wrap(err, "artifact processor: failed to merge provenance") + } + newProv.AddPostprocess(postprocess) + newProv.Frontend = oldProv.Frontend + newProv.Args = maps.Clone(oldProv.Args) + } + + res.Provenance = postRes.Provenance + res.Provenance.Ref = newProv + res.Provenance.Refs = nil + return nil +} + +func normalizeArtifactResultMetadata(meta map[string][]byte) error { + if meta == nil { + return nil + } + + platformsBytes, ok := meta[exptypes.ExporterPlatformsKey] + if !ok { + return nil + } + + var platforms exptypes.Platforms + if err := json.Unmarshal(platformsBytes, &platforms); err != nil { + return errors.Wrap(err, "artifact processor: failed to parse platforms metadata") + } + if len(platforms.Platforms) != 1 { + return errors.Errorf("artifact processor: expected exactly one platform mapping, got %d", len(platforms.Platforms)) + } + + platformID := platforms.Platforms[0].ID + for _, key := range exptypes.KnownRefMetadataKeys { + if _, ok := meta[key]; ok { + continue + } + if v, ok := meta[key+"/"+platformID]; ok { + meta[key] = v + } + } + return nil +} diff --git a/solver/llbsolver/proc/artifact_test.go b/solver/llbsolver/proc/artifact_test.go new file mode 100644 index 000000000000..311ed3f3065c --- /dev/null +++ b/solver/llbsolver/proc/artifact_test.go @@ -0,0 +1,69 @@ +package proc + +import ( + "encoding/json" + "testing" + + "github.com/containerd/platforms" + "github.com/moby/buildkit/exporter/containerimage/exptypes" + "github.com/moby/buildkit/frontend" + artifactfrontend "github.com/moby/buildkit/frontend/artifact" + "github.com/moby/buildkit/solver/llbsolver" + "github.com/moby/buildkit/solver/llbsolver/provenance" + "github.com/stretchr/testify/require" +) + +func TestUpdateArtifactProvenanceAddsPostprocess(t *testing.T) { + res := &llbsolver.Result{ + Result: &frontend.Result{}, + Provenance: &provenance.Result{ + Ref: &provenance.Capture{ + Frontend: "dockerfile.v0", + Args: map[string]string{ + "target": "release", + }, + }, + }, + } + + postRes := &llbsolver.Result{ + Result: &frontend.Result{}, + Provenance: &provenance.Result{ + Ref: &provenance.Capture{ + Frontend: artifactfrontend.Name, + Args: map[string]string{ + "type": "artifact", + }, + }, + }, + } + + require.NoError(t, updateArtifactProvenance(res, postRes)) + require.NotNil(t, res.Provenance) + require.NotNil(t, res.Provenance.Ref) + require.Equal(t, "dockerfile.v0", res.Provenance.Ref.Frontend) + require.Equal(t, "release", res.Provenance.Ref.Args["target"]) + require.Len(t, res.Provenance.Ref.Postprocess, 1) + require.Equal(t, artifactfrontend.Name, res.Provenance.Ref.Postprocess[0].Frontend) + require.Equal(t, "artifact", res.Provenance.Ref.Postprocess[0].Args["type"]) +} + +func TestNormalizeArtifactResultMetadataPreservesSinglePlatformMapping(t *testing.T) { + ps := exptypes.Platforms{ + Platforms: []exptypes.Platform{{ + ID: platforms.Format(platforms.MustParse("linux/arm64")), + Platform: platforms.MustParse("linux/arm64"), + }}, + } + psJSON, err := json.Marshal(ps) + require.NoError(t, err) + + meta := map[string][]byte{ + exptypes.ExporterPlatformsKey: psJSON, + exptypes.ExporterImageConfigKey + "/" + ps.Platforms[0].ID: []byte("config"), + } + + require.NoError(t, normalizeArtifactResultMetadata(meta)) + require.Equal(t, []byte("config"), meta[exptypes.ExporterImageConfigKey]) + require.Equal(t, psJSON, meta[exptypes.ExporterPlatformsKey]) +} diff --git a/solver/llbsolver/provenance/capture.go b/solver/llbsolver/provenance/capture.go index 800304409681..5dfc44c1b292 100644 --- a/solver/llbsolver/provenance/capture.go +++ b/solver/llbsolver/provenance/capture.go @@ -2,6 +2,7 @@ package provenance import ( "cmp" + "maps" "slices" distreference "github.com/distribution/reference" @@ -17,6 +18,7 @@ type Result = result.Result[*Capture] type Capture struct { Frontend string Args map[string]string + Postprocess []provenancetypes.Parameters Sources provenancetypes.Sources Secrets []provenancetypes.Secret SSH []provenancetypes.SSH @@ -50,6 +52,9 @@ func (c *Capture) Merge(c2 *Capture) error { for _, s := range c2.SSH { c.AddSSH(s) } + for _, p := range c2.Postprocess { + c.AddPostprocess(p) + } if c2.NetworkAccess { c.NetworkAccess = true } @@ -197,6 +202,38 @@ func (c *Capture) AddSSH(s provenancetypes.SSH) { c.SSH = append(c.SSH, s) } +func (c *Capture) Parameters() provenancetypes.Parameters { + p := provenancetypes.Parameters{ + Frontend: c.Frontend, + Args: maps.Clone(c.Args), + } + for _, s := range c.Secrets { + s2 := s + p.Secrets = append(p.Secrets, &s2) + } + for _, s := range c.SSH { + s2 := s + p.SSH = append(p.SSH, &s2) + } + for _, s := range c.Sources.Local { + s2 := s + p.Locals = append(p.Locals, &s2) + } + return p +} + +func (c *Capture) AddPostprocess(p provenancetypes.Parameters) { + if p.Frontend == "" { + return + } + for _, existing := range c.Postprocess { + if equalParameters(existing, p) { + return + } + } + c.Postprocess = append(c.Postprocess, cloneParameters(p)) +} + func (c *Capture) AddSamples(dgst digest.Digest, samples *resourcestypes.Samples) { if c.Samples == nil { c.Samples = map[digest.Digest]*resourcestypes.Samples{} @@ -204,6 +241,84 @@ func (c *Capture) AddSamples(dgst digest.Digest, samples *resourcestypes.Samples c.Samples[dgst] = samples } +func cloneParameters(p provenancetypes.Parameters) provenancetypes.Parameters { + out := provenancetypes.Parameters{ + Frontend: p.Frontend, + Args: maps.Clone(p.Args), + } + for _, s := range p.Secrets { + if s == nil { + out.Secrets = append(out.Secrets, nil) + continue + } + s2 := *s + out.Secrets = append(out.Secrets, &s2) + } + for _, s := range p.SSH { + if s == nil { + out.SSH = append(out.SSH, nil) + continue + } + s2 := *s + out.SSH = append(out.SSH, &s2) + } + for _, l := range p.Locals { + if l == nil { + out.Locals = append(out.Locals, nil) + continue + } + l2 := *l + out.Locals = append(out.Locals, &l2) + } + return out +} + +func equalParameters(a, b provenancetypes.Parameters) bool { + if a.Frontend != b.Frontend || !maps.Equal(a.Args, b.Args) { + return false + } + if len(a.Secrets) != len(b.Secrets) || len(a.SSH) != len(b.SSH) || len(a.Locals) != len(b.Locals) { + return false + } + for i := range a.Secrets { + if !equalSecret(a.Secrets[i], b.Secrets[i]) { + return false + } + } + for i := range a.SSH { + if !equalSSH(a.SSH[i], b.SSH[i]) { + return false + } + } + for i := range a.Locals { + if !equalLocal(a.Locals[i], b.Locals[i]) { + return false + } + } + return true +} + +func equalSecret(a, b *provenancetypes.Secret) bool { + if a == nil || b == nil { + return a == b + } + return a.ID == b.ID && a.Optional == b.Optional +} + +func equalSSH(a, b *provenancetypes.SSH) bool { + if a == nil || b == nil { + return a == b + } + return a.ID == b.ID && a.Optional == b.Optional +} + +func equalLocal(a, b *provenancetypes.LocalSource) bool { + if a == nil || b == nil { + return a == b + } + return a.Name == b.Name +} + func parseRefName(s string) (distreference.Named, string, error) { ref, err := distreference.ParseNormalizedNamed(s) if err != nil { diff --git a/solver/llbsolver/provenance/predicate.go b/solver/llbsolver/provenance/predicate.go index d82d04cb0474..f3ca2cf0d48b 100644 --- a/solver/llbsolver/provenance/predicate.go +++ b/solver/llbsolver/provenance/predicate.go @@ -188,6 +188,9 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error) internal := provenancetypes.ProvenanceInternalParametersSLSA1{} internal.BuilderPlatform = platforms.Format(platforms.Normalize(platforms.DefaultSpec())) + if len(c.Postprocess) > 0 { + internal.Postprocess = cloneParametersSlice(c.Postprocess) + } req := provenancetypes.Parameters{} req.Frontend = c.Frontend @@ -245,6 +248,17 @@ func NewPredicate(c *Capture) (*provenancetypes.ProvenancePredicateSLSA1, error) return pr, nil } +func cloneParametersSlice(src []provenancetypes.Parameters) []provenancetypes.Parameters { + if len(src) == 0 { + return nil + } + out := make([]provenancetypes.Parameters, 0, len(src)) + for _, p := range src { + out = append(out, cloneParameters(p)) + } + return out +} + func FilterArgs(m map[string]string) map[string]string { var hostSpecificArgs = map[string]struct{}{ "cgroup-parent": {}, diff --git a/solver/llbsolver/provenance/types/types.go b/solver/llbsolver/provenance/types/types.go index c077f07158ca..c2202e920b25 100644 --- a/solver/llbsolver/provenance/types/types.go +++ b/solver/llbsolver/provenance/types/types.go @@ -160,6 +160,7 @@ type ProvenanceConfigSourceSLSA1 struct { type ProvenanceInternalParametersSLSA1 struct { BuildConfig *BuildConfig `json:"buildConfig,omitempty"` BuilderPlatform string `json:"builderPlatform"` + Postprocess []Parameters `json:"postprocess,omitempty"` ProvenanceCustomEnv } @@ -182,7 +183,8 @@ type Parameters struct { } type Environment struct { - Platform string `json:"platform"` + Platform string `json:"platform"` + Postprocess []Parameters `json:"postprocess,omitempty"` ProvenanceCustomEnv } @@ -246,6 +248,7 @@ func (p *ProvenancePredicateSLSA1) ConvertToSLSA02() *ProvenancePredicateSLSA02 Parameters: p.BuildDefinition.ExternalParameters.Request, Environment: Environment{ Platform: p.BuildDefinition.InternalParameters.BuilderPlatform, + Postprocess: p.BuildDefinition.InternalParameters.Postprocess, ProvenanceCustomEnv: p.BuildDefinition.InternalParameters.ProvenanceCustomEnv, }, }, @@ -280,6 +283,7 @@ func (p *ProvenancePredicateSLSA02) ConvertToSLSA1() *ProvenancePredicateSLSA1 { InternalParameters: ProvenanceInternalParametersSLSA1{ BuildConfig: p.BuildConfig, BuilderPlatform: p.Invocation.Environment.Platform, + Postprocess: p.Invocation.Environment.Postprocess, ProvenanceCustomEnv: p.Invocation.Environment.ProvenanceCustomEnv, }, } diff --git a/solver/llbsolver/provenance/types/types_test.go b/solver/llbsolver/provenance/types/types_test.go index 5da999944775..30e22eeba53b 100644 --- a/solver/llbsolver/provenance/types/types_test.go +++ b/solver/llbsolver/provenance/types/types_test.go @@ -16,6 +16,12 @@ func TestMarsalBuildDefinitionSLSA1(t *testing.T) { }, "internalParameters": { "builderPlatform": "linux/amd64", + "postprocess": [{ + "frontend": "artifact.v0", + "args": { + "type": "oci" + } + }], "foo": "bar", "abc": 123, "def": {"one": 1} @@ -28,6 +34,9 @@ func TestMarsalBuildDefinitionSLSA1(t *testing.T) { require.Equal(t, "btype1", def.BuildType) require.Equal(t, "linux/amd64", def.InternalParameters.BuilderPlatform) + require.Len(t, def.InternalParameters.Postprocess, 1) + require.Equal(t, "artifact.v0", def.InternalParameters.Postprocess[0].Frontend) + require.Equal(t, "oci", def.InternalParameters.Postprocess[0].Args["type"]) require.Equal(t, "bar", def.InternalParameters.ProvenanceCustomEnv["foo"]) require.InEpsilon(t, float64(123), def.InternalParameters.ProvenanceCustomEnv["abc"], 0.001) require.Equal(t, map[string]any{"one": float64(1)}, def.InternalParameters.ProvenanceCustomEnv["def"]) @@ -48,6 +57,9 @@ func TestMarshalInvocation(t *testing.T) { }, "environment": { "platform": "linux/amd64", + "postprocess": [{ + "frontend": "artifact.v0" + }], "buildkit": "v0.10.3", "custom": { "foo": "bar" @@ -63,6 +75,8 @@ func TestMarshalInvocation(t *testing.T) { require.Equal(t, "git+https://github.com/example/repo.git", inv.ConfigSource.URI) require.Equal(t, "dockerfile.v0", inv.Parameters.Frontend) require.Equal(t, "linux/amd64", inv.Environment.Platform) + require.Len(t, inv.Environment.Postprocess, 1) + require.Equal(t, "artifact.v0", inv.Environment.Postprocess[0].Frontend) require.Equal(t, "v0.10.3", inv.Environment.ProvenanceCustomEnv["buildkit"]) require.Equal(t, "bar", inv.Environment.ProvenanceCustomEnv["custom"].(map[string]any)["foo"]) require.Equal(t, []any{float64(1), float64(2), float64(3)}, inv.Environment.ProvenanceCustomEnv["bar"]) diff --git a/solver/llbsolver/solver.go b/solver/llbsolver/solver.go index c1f6317fe513..eab183b178e9 100644 --- a/solver/llbsolver/solver.go +++ b/solver/llbsolver/solver.go @@ -152,6 +152,32 @@ func (s *Solver) Bridge(b solver.Builder) frontend.FrontendLLBBridge { return s.bridge(b) } +// SolvePostprocessor runs an additional solve through the frontend pipeline so +// the converted result is tracked like a regular frontend solve before export. +func (s *Solver) SolvePostprocessor(ctx context.Context, j *solver.Job, req frontend.SolveRequest) (*Result, error) { + br := s.bridge(j) + + res, err := br.Solve(ctx, req, j.SessionID) + if err != nil { + return nil, err + } + + eg, ctx2 := errgroup.WithContext(ctx) + res.EachRef(func(ref solver.ResultProxy) error { + eg.Go(func() error { + _, err := ref.Result(ctx2) + return err + }) + return nil + }) + if err := eg.Wait(); err != nil { + return nil, err + } + + return addProvenanceToResult(res, br) +} + + func (s *Solver) Solve(ctx context.Context, id string, sessionID string, req frontend.SolveRequest, exp ExporterRequest, ent []entitlements.Entitlement, post []Processor, internal bool, srcPol *spb.Policy, policySession string) (_ *client.SolveResponse, err error) { j, err := s.solver.NewJob(id) if err != nil { diff --git a/util/push/push.go b/util/push/push.go index 77788da20cb4..a336f194e683 100644 --- a/util/push/push.go +++ b/util/push/push.go @@ -255,7 +255,7 @@ func childrenHandler(provider content.Provider) images.HandlerFunc { // childless data types. return nil, nil default: - bklog.G(ctx).Warnf("encountered unknown type %v; children may not be fetched", desc.MediaType) + bklog.G(ctx).Debugf("encountered unknown type %v; children may not be fetched", desc.MediaType) } return descs, nil