c8d/push: Push multi-platform images

daemon/containerd/image.go

@@ -10,15 +10,19 @@ import (
 	"github.com/containerd/containerd/content"
 	cerrdefs "github.com/containerd/containerd/errdefs"
 	containerdimages "github.com/containerd/containerd/images"
+	"github.com/containerd/containerd/images/converter"
+	"github.com/containerd/containerd/platforms"
 	"github.com/docker/distribution/reference"
 	containertypes "github.com/docker/docker/api/types/container"
 	imagetype "github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/errdefs"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
+	"github.com/google/uuid"
 	"github.com/opencontainers/go-digest"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 )
 
 var shortID = regexp.MustCompile(`^([a-f0-9]{4,64})$`)
@@ -258,3 +262,92 @@ func (i *ImageService) resolveImageName(ctx context.Context, refOrID string) (oc
 
 	return img.Target, namedRef, nil
 }
+
+// createFullestImage converts the given manifest list to a fullest possible with the current store content.
+// If all manifests blobs are present in the content store, then no conversion is performed and it returns nil.
+// Otherwise it creates a new image which targets a manifest list referencing only manifests with present blobs.
+func (i *ImageService) createFullestImage(ctx context.Context, img containerdimages.Image) (*containerdimages.Image, error) {
+	if containerdimages.IsIndexType(img.Target.MediaType) {
+		// Check which platform manifests we have blobs for.
+		presentPlatforms, missingPlatforms, err := i.checkPresentPlatforms(ctx, img.Target)
+
+		// If we have all the manifests, no need to convert the image.
+		if len(missingPlatforms) == 0 {
+			return nil, nil
+		}
+
+		// Create a new manifest list which contains only the manifests we have in store.
+		srcRef := img.Name
+		targetRef := srcRef + "-tmp-" + uuid.NewString()
+		for _, p := range presentPlatforms {
+			targetRef += "-" + platforms.Format(p)
+		}
+
+		newImg, err := converter.Convert(ctx, i.client, targetRef, srcRef,
+			converter.WithPlatform(platforms.Any(presentPlatforms...)))
+		if err != nil {
+			return nil, err
+		}
+		return newImg, nil
+	}
+
+	return nil, nil
+}
+
+// checkPresentPlatforms returns the platforms for which the image targeting an index/manifest list
+// has all the blobs present in the content store.
+// Both present and missing platforms list returned if there wasn't any error.
+func (i *ImageService) checkPresentPlatforms(ctx context.Context, index ocispec.Descriptor) ([]ocispec.Platform, []ocispec.Platform, error) {
+	if !containerdimages.IsIndexType(index.MediaType) {
+		return nil, nil, errors.New("not an index/manifest list")
+	}
+
+	store := i.client.ContentStore()
+
+	children, err := containerdimages.Children(ctx, store, index)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	missingPlatforms := []ocispec.Platform{}
+	presentPlatforms := []ocispec.Platform{}
+	for _, child := range children {
+		if containerdimages.IsManifestType(child.MediaType) {
+			missing := false
+
+			_, err := store.ReaderAt(ctx, child)
+			if cerrdefs.IsNotFound(err) {
+				missing = true
+			} else if err != nil {
+				return nil, nil, err
+			}
+
+			// If the manifest is not missing, also check for its blobs.
+			if !missing {
+				manifestChildren, err := containerdimages.Children(ctx, store, child)
+				if err != nil {
+					return nil, nil, err
+				}
+
+				// If any blob is missing, mark the manifest as missing.
+				for _, manifestChild := range manifestChildren {
+					_, err := store.ReaderAt(ctx, manifestChild)
+					if cerrdefs.IsNotFound(err) {
+						missing = true
+						break
+					} else if err != nil {
+						return nil, nil, err
+					}
+				}
+			}
+
+			if missing {
+				missingPlatforms = append(missingPlatforms, *child.Platform)
+				logrus.WithField("digest", child.Digest.String()).Debug("missing blob")
+			} else {
+				presentPlatforms = append(presentPlatforms, *child.Platform)
+			}
+		}
+	}
+	return presentPlatforms, missingPlatforms, nil
+}

daemon/containerd/image_exporter.go

@@ -5,13 +5,10 @@ import (
 	"io"
 
 	"github.com/containerd/containerd"
-	cerrdefs "github.com/containerd/containerd/errdefs"
 	containerdimages "github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/images/archive"
-	"github.com/containerd/containerd/images/converter"
 	"github.com/containerd/containerd/platforms"
 	"github.com/docker/distribution/reference"
-	"github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -58,71 +55,32 @@ func (i *ImageService) ExportImage(ctx context.Context, names []string, outStrea
 		archive.WithSkipNonDistributableBlobs(),
 	}
 
-	for _, imageRef := range names {
-		var err error
-		opts, err = i.appendImageForExport(ctx, opts, imageRef)
+	is := i.client.ImageService()
+
+	for _, name := range names {
+		ref, err := reference.ParseDockerRef(name)
 		if err != nil {
 			return err
 		}
-	}
-
-	return i.client.Export(ctx, outStream, opts...)
-}
 
-func (i *ImageService) appendImageForExport(ctx context.Context, opts []archive.ExportOpt, name string) ([]archive.ExportOpt, error) {
-	ref, err := reference.ParseDockerRef(name)
-	if err != nil {
-		return opts, err
-	}
-
-	is := i.client.ImageService()
-
-	img, err := is.Get(ctx, ref.String())
-	if err != nil {
-		return opts, err
-	}
-
-	store := i.client.ContentStore()
-
-	if containerdimages.IsIndexType(img.Target.MediaType) {
-		children, err := containerdimages.Children(ctx, store, img.Target)
+		img, err := is.Get(ctx, ref.String())
 		if err != nil {
-			return opts, err
+			return err
 		}
 
-		// Check which platform manifests we have blobs for.
-		missingPlatforms := []v1.Platform{}
-		presentPlatforms := []v1.Platform{}
-		for _, child := range children {
-			if containerdimages.IsManifestType(child.MediaType) {
-				_, err := store.ReaderAt(ctx, child)
-				if cerrdefs.IsNotFound(err) {
-					missingPlatforms = append(missingPlatforms, *child.Platform)
-					logrus.WithField("digest", child.Digest.String()).Debug("missing blob, not exporting")
-					continue
-				} else if err != nil {
-					return opts, err
-				}
-				presentPlatforms = append(presentPlatforms, *child.Platform)
-			}
+		converted, err := i.createFullestImage(ctx, img)
+		if err != nil {
+			return err
 		}
 
-		// If we have all the manifests, just export the original index.
-		if len(missingPlatforms) == 0 {
-			return append(opts, archive.WithImage(is, img.Name)), nil
+		if converted != nil {
+			opts = append(opts, archive.WithImage(is, converted.Name))
+			defer is.Delete(context.Background(), converted.Name, containerdimages.SynchronousDelete())
+			continue
 		}
 
-		// Create a new manifest which contains only the manifests we have in store.
-		srcRef := ref.String()
-		targetRef := srcRef + "-tmp-export"
-		newImg, err := converter.Convert(ctx, i.client, targetRef, srcRef,
-			converter.WithPlatform(platforms.Any(presentPlatforms...)))
-		if err != nil {
-			return opts, err
-		}
-		defer i.client.ImageService().Delete(ctx, newImg.Name, containerdimages.SynchronousDelete())
-		return append(opts, archive.WithManifest(newImg.Target, srcRef)), nil
+		opts = append(opts, archive.WithImage(is, img.Name))
 	}
 
-	return append(opts, archive.WithImage(is, img.Name)), nil
+	return i.client.Export(ctx, outStream, opts...)
 }

daemon/containerd/image_push.go

@@ -11,6 +11,7 @@ import (
 	"github.com/containerd/containerd/remotes"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types/registry"
+	"github.com/google/uuid"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -19,7 +20,7 @@ import (
 // PushImage initiates a push operation on the repository named localName.
 func (i *ImageService) PushImage(ctx context.Context, image, tag string, metaHeaders map[string][]string, authConfig *registry.AuthConfig, outStream io.Writer) error {
 	// TODO: Pass this from user?
-	platformMatcher := platforms.DefaultStrict()
+	platformMatcher := platforms.All
 
 	ref, err := reference.ParseNormalizedNamed(image)
 	if err != nil {
@@ -43,17 +44,29 @@ func (i *ImageService) PushImage(ctx context.Context, image, tag string, metaHea
 
 	target := img.Target
 
-	// Create a temporary image which is stripped from content that references other platforms.
-	// We or the remote may not have them and referencing them will end with an error.
-	if platformMatcher != platforms.All {
-		tmpRef := ref.String() + "-tmp-platformspecific"
-		platformImg, err := converter.Convert(ctx, i.client, tmpRef, ref.String(), converter.WithPlatform(platformMatcher))
-		if err != nil {
-			return errors.Wrap(err, "Failed to convert image to platform specific")
-		}
+	if containerdimages.IsIndexType(target.MediaType) {
+		if platformMatcher == platforms.All {
+			// Create an image with manifest list that references only present blobs.
+			fullest, err := i.createFullestImage(ctx, img)
+			if err != nil {
+				return err
+			}
+			if fullest != nil {
+				target = fullest.Target
+				defer i.client.ImageService().Delete(ctx, fullest.Name, containerdimages.SynchronousDelete())
+			}
+		} else {
+			// If user requests to push a specific platform, convert the index to one that
+			// doesn't reference manifest of other platforms.
+			tmpRef := ref.String() + "-tmp-" + uuid.NewString()
+			platformImg, err := converter.Convert(ctx, i.client, tmpRef, ref.String(), converter.WithPlatform(platformMatcher))
+			if err != nil {
+				return errors.Wrap(err, "failed to convert image to platform specific")
+			}
 
-		target = platformImg.Target
-		defer i.client.ImageService().Delete(ctx, platformImg.Name, containerdimages.SynchronousDelete())
+			target = platformImg.Target
+			defer i.client.ImageService().Delete(ctx, platformImg.Name, containerdimages.SynchronousDelete())
+		}
 	}
 
 	jobs := newJobs()