diff --git a/daemon/containerd/image.go b/daemon/containerd/image.go index 37de1f5553c0a..47964d7a08462 100644 --- a/daemon/containerd/image.go +++ b/daemon/containerd/image.go @@ -10,15 +10,19 @@ import ( "github.com/containerd/containerd/content" cerrdefs "github.com/containerd/containerd/errdefs" containerdimages "github.com/containerd/containerd/images" + "github.com/containerd/containerd/images/converter" + "github.com/containerd/containerd/platforms" "github.com/docker/distribution/reference" containertypes "github.com/docker/docker/api/types/container" imagetype "github.com/docker/docker/api/types/image" "github.com/docker/docker/errdefs" "github.com/docker/docker/image" "github.com/docker/docker/layer" + "github.com/google/uuid" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) var shortID = regexp.MustCompile(`^([a-f0-9]{4,64})$`) @@ -258,3 +262,92 @@ func (i *ImageService) resolveImageName(ctx context.Context, refOrID string) (oc return img.Target, namedRef, nil } + +// createFullestImage converts the given manifest list to a fullest possible with the current store content. +// If all manifests blobs are present in the content store, then no conversion is performed and it returns nil. +// Otherwise it creates a new image which targets a manifest list referencing only manifests with present blobs. +func (i *ImageService) createFullestImage(ctx context.Context, img containerdimages.Image) (*containerdimages.Image, error) { + if containerdimages.IsIndexType(img.Target.MediaType) { + // Check which platform manifests we have blobs for. + presentPlatforms, missingPlatforms, err := i.checkPresentPlatforms(ctx, img.Target) + + // If we have all the manifests, no need to convert the image. + if len(missingPlatforms) == 0 { + return nil, nil + } + + // Create a new manifest list which contains only the manifests we have in store. + srcRef := img.Name + targetRef := srcRef + "-tmp-" + uuid.NewString() + for _, p := range presentPlatforms { + targetRef += "-" + platforms.Format(p) + } + + newImg, err := converter.Convert(ctx, i.client, targetRef, srcRef, + converter.WithPlatform(platforms.Any(presentPlatforms...))) + if err != nil { + return nil, err + } + return newImg, nil + } + + return nil, nil +} + +// checkPresentPlatforms returns the platforms for which the image targeting an index/manifest list +// has all the blobs present in the content store. +// Both present and missing platforms list returned if there wasn't any error. +func (i *ImageService) checkPresentPlatforms(ctx context.Context, index ocispec.Descriptor) ([]ocispec.Platform, []ocispec.Platform, error) { + if !containerdimages.IsIndexType(index.MediaType) { + return nil, nil, errors.New("not an index/manifest list") + } + + store := i.client.ContentStore() + + children, err := containerdimages.Children(ctx, store, index) + if err != nil { + return nil, nil, err + } + + missingPlatforms := []ocispec.Platform{} + presentPlatforms := []ocispec.Platform{} + for _, child := range children { + if containerdimages.IsManifestType(child.MediaType) { + missing := false + + _, err := store.ReaderAt(ctx, child) + if cerrdefs.IsNotFound(err) { + missing = true + } else if err != nil { + return nil, nil, err + } + + // If the manifest is not missing, also check for its blobs. + if !missing { + manifestChildren, err := containerdimages.Children(ctx, store, child) + if err != nil { + return nil, nil, err + } + + // If any blob is missing, mark the manifest as missing. + for _, manifestChild := range manifestChildren { + _, err := store.ReaderAt(ctx, manifestChild) + if cerrdefs.IsNotFound(err) { + missing = true + break + } else if err != nil { + return nil, nil, err + } + } + } + + if missing { + missingPlatforms = append(missingPlatforms, *child.Platform) + logrus.WithField("digest", child.Digest.String()).Debug("missing blob") + } else { + presentPlatforms = append(presentPlatforms, *child.Platform) + } + } + } + return presentPlatforms, missingPlatforms, nil +} diff --git a/daemon/containerd/image_exporter.go b/daemon/containerd/image_exporter.go index 679f3b1c2ec24..38b7bf7a9ef63 100644 --- a/daemon/containerd/image_exporter.go +++ b/daemon/containerd/image_exporter.go @@ -5,13 +5,10 @@ import ( "io" "github.com/containerd/containerd" - cerrdefs "github.com/containerd/containerd/errdefs" containerdimages "github.com/containerd/containerd/images" "github.com/containerd/containerd/images/archive" - "github.com/containerd/containerd/images/converter" "github.com/containerd/containerd/platforms" "github.com/docker/distribution/reference" - "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -58,71 +55,32 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, outStrea archive.WithSkipNonDistributableBlobs(), } - for _, imageRef := range names { - var err error - opts, err = i.appendImageForExport(ctx, opts, imageRef) + is := i.client.ImageService() + + for _, name := range names { + ref, err := reference.ParseDockerRef(name) if err != nil { return err } - } - - return i.client.Export(ctx, outStream, opts...) -} -func (i *ImageService) appendImageForExport(ctx context.Context, opts []archive.ExportOpt, name string) ([]archive.ExportOpt, error) { - ref, err := reference.ParseDockerRef(name) - if err != nil { - return opts, err - } - - is := i.client.ImageService() - - img, err := is.Get(ctx, ref.String()) - if err != nil { - return opts, err - } - - store := i.client.ContentStore() - - if containerdimages.IsIndexType(img.Target.MediaType) { - children, err := containerdimages.Children(ctx, store, img.Target) + img, err := is.Get(ctx, ref.String()) if err != nil { - return opts, err + return err } - // Check which platform manifests we have blobs for. - missingPlatforms := []v1.Platform{} - presentPlatforms := []v1.Platform{} - for _, child := range children { - if containerdimages.IsManifestType(child.MediaType) { - _, err := store.ReaderAt(ctx, child) - if cerrdefs.IsNotFound(err) { - missingPlatforms = append(missingPlatforms, *child.Platform) - logrus.WithField("digest", child.Digest.String()).Debug("missing blob, not exporting") - continue - } else if err != nil { - return opts, err - } - presentPlatforms = append(presentPlatforms, *child.Platform) - } + converted, err := i.createFullestImage(ctx, img) + if err != nil { + return err } - // If we have all the manifests, just export the original index. - if len(missingPlatforms) == 0 { - return append(opts, archive.WithImage(is, img.Name)), nil + if converted != nil { + opts = append(opts, archive.WithImage(is, converted.Name)) + defer is.Delete(context.Background(), converted.Name, containerdimages.SynchronousDelete()) + continue } - // Create a new manifest which contains only the manifests we have in store. - srcRef := ref.String() - targetRef := srcRef + "-tmp-export" - newImg, err := converter.Convert(ctx, i.client, targetRef, srcRef, - converter.WithPlatform(platforms.Any(presentPlatforms...))) - if err != nil { - return opts, err - } - defer i.client.ImageService().Delete(ctx, newImg.Name, containerdimages.SynchronousDelete()) - return append(opts, archive.WithManifest(newImg.Target, srcRef)), nil + opts = append(opts, archive.WithImage(is, img.Name)) } - return append(opts, archive.WithImage(is, img.Name)), nil + return i.client.Export(ctx, outStream, opts...) } diff --git a/daemon/containerd/image_push.go b/daemon/containerd/image_push.go index 2a1b3451871ce..e7abc0c81a96c 100644 --- a/daemon/containerd/image_push.go +++ b/daemon/containerd/image_push.go @@ -11,6 +11,7 @@ import ( "github.com/containerd/containerd/remotes" "github.com/docker/distribution/reference" "github.com/docker/docker/api/types/registry" + "github.com/google/uuid" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -19,7 +20,7 @@ import ( // PushImage initiates a push operation on the repository named localName. func (i *ImageService) PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error { // TODO: Pass this from user? - platformMatcher := platforms.DefaultStrict() + platformMatcher := platforms.All ref, err := reference.ParseNormalizedNamed(image) if err != nil { @@ -43,17 +44,29 @@ func (i *ImageService) PushImage(ctx context.Context, image, tag string, metaHea target := img.Target - // Create a temporary image which is stripped from content that references other platforms. - // We or the remote may not have them and referencing them will end with an error. - if platformMatcher != platforms.All { - tmpRef := ref.String() + "-tmp-platformspecific" - platformImg, err := converter.Convert(ctx, i.client, tmpRef, ref.String(), converter.WithPlatform(platformMatcher)) - if err != nil { - return errors.Wrap(err, "Failed to convert image to platform specific") - } + if containerdimages.IsIndexType(target.MediaType) { + if platformMatcher == platforms.All { + // Create an image with manifest list that references only present blobs. + fullest, err := i.createFullestImage(ctx, img) + if err != nil { + return err + } + if fullest != nil { + target = fullest.Target + defer i.client.ImageService().Delete(ctx, fullest.Name, containerdimages.SynchronousDelete()) + } + } else { + // If user requests to push a specific platform, convert the index to one that + // doesn't reference manifest of other platforms. + tmpRef := ref.String() + "-tmp-" + uuid.NewString() + platformImg, err := converter.Convert(ctx, i.client, tmpRef, ref.String(), converter.WithPlatform(platformMatcher)) + if err != nil { + return errors.Wrap(err, "failed to convert image to platform specific") + } - target = platformImg.Target - defer i.client.ImageService().Delete(ctx, platformImg.Name, containerdimages.SynchronousDelete()) + target = platformImg.Target + defer i.client.ImageService().Delete(ctx, platformImg.Name, containerdimages.SynchronousDelete()) + } } jobs := newJobs()