Make the snapshotter configurable - review suggestions / ideas

daemon/config/config.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/containerd/containerd"
 	"github.com/docker/docker/opts"
 	"github.com/docker/docker/pkg/authorization"
 	"github.com/docker/docker/registry"
@@ -51,6 +52,9 @@ const (
 	// DefaultPluginNamespace is the name of the default containerd namespace used for plugins.
 	DefaultPluginNamespace = "plugins.moby"
 
+	// DefaultContainerdSnapshotter is the name of the default containerd snapshotter used for creating container root fs
+	DefaultContainerdSnapshotter = containerd.DefaultSnapshotter
+
 	// LinuxV2RuntimeName is the runtime used to specify the containerd v2 runc shim
 	LinuxV2RuntimeName = "io.containerd.runc.v2"
 

daemon/containerd/image_exporter.go

@@ -48,14 +48,14 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outSt
 	for _, img := range imgs {
 		platformImg := containerd.NewImageWithPlatform(i.client, img, platform)
 
-		unpacked, err := platformImg.IsUnpacked(ctx, containerd.DefaultSnapshotter)
+		unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter)
 		if err != nil {
 			logrus.WithError(err).WithField("image", img.Name).Error("IsUnpacked failed")
 			continue
 		}
 
 		if !unpacked {
-			err := platformImg.Unpack(ctx, containerd.DefaultSnapshotter)
+			err := platformImg.Unpack(ctx, i.snapshotter)
 			if err != nil {
 				logrus.WithError(err).WithField("image", img.Name).Error("Failed to unpack image")
 				return errors.Wrapf(err, "Failed to unpack image")

daemon/containerd/image_list.go

@@ -37,7 +37,7 @@ func (i *ImageService) Images(ctx context.Context, opts types.ImageListOptions)
 		return nil, err
 	}
 
-	snapshotter := i.client.SnapshotService(containerd.DefaultSnapshotter)
+	snapshotter := i.client.SnapshotService(i.snapshotter)
 	sizeCache := make(map[digest.Digest]int64)
 	snapshotSizeFn := func(d digest.Digest) (int64, error) {
 		if s, ok := sizeCache[d]; ok {

daemon/containerd/image_pull.go

@@ -62,13 +62,13 @@ func (i *ImageService) PullImage(ctx context.Context, image, tagOrDigest string,
 		return err
 	}
 
-	unpacked, err := img.IsUnpacked(ctx, containerd.DefaultSnapshotter)
+	unpacked, err := img.IsUnpacked(ctx, i.snapshotter)
 	if err != nil {
 		return err
 	}
 
 	if !unpacked {
-		if err := img.Unpack(ctx, containerd.DefaultSnapshotter); err != nil {
+		if err := img.Unpack(ctx, i.snapshotter); err != nil {
 			return err
 		}
 	}

daemon/containerd/service.go

@@ -19,16 +19,18 @@ import (
 
 // ImageService implements daemon.ImageService
 type ImageService struct {
-	client     *containerd.Client
-	usage      singleflight.Group
-	containers container.Store
+	client      *containerd.Client
+	usage       singleflight.Group
+	containers  container.Store
+	snapshotter string
 }
 
 // NewService creates a new ImageService.
-func NewService(c *containerd.Client, containers container.Store) *ImageService {
+func NewService(c *containerd.Client, containers container.Store, snapshotter string) *ImageService {
 	return &ImageService{
-		client:     c,
-		containers: containers,
+		client:      c,
+		containers:  containers,
+		snapshotter: snapshotter,
 	}
 }
 
@@ -92,7 +94,7 @@ func (i *ImageService) Cleanup() error {
 // - newContainer
 // - to report an error in Daemon.Mount(container)
 func (i *ImageService) GraphDriverName() string {
-	return "containerd-snapshotter"
+	return i.snapshotter
 }
 
 // ReleaseLayer releases a layer allowing it to be removed
@@ -106,7 +108,7 @@ func (i *ImageService) ReleaseLayer(rwlayer layer.RWLayer) error {
 func (i *ImageService) LayerDiskUsage(ctx context.Context) (int64, error) {
 	ch := i.usage.DoChan("LayerDiskUsage", func() (interface{}, error) {
 		var allLayersSize int64
-		snapshotter := i.client.SnapshotService(containerd.DefaultSnapshotter)
+		snapshotter := i.client.SnapshotService(i.snapshotter)
 		snapshotter.Walk(ctx, func(ctx context.Context, info snapshots.Info) error {
 			usage, err := snapshotter.Usage(ctx, info.Name)
 			if err != nil {
@@ -167,7 +169,7 @@ func (i *ImageService) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer)
 
 // GetContainerLayerSize returns the real size & virtual size of the container.
 func (i *ImageService) GetContainerLayerSize(ctx context.Context, containerID string) (int64, int64, error) {
-	snapshotter := i.client.SnapshotService(containerd.DefaultSnapshotter)
+	snapshotter := i.client.SnapshotService(i.snapshotter)
 	sizeCache := make(map[digest.Digest]int64)
 	snapshotSizeFn := func(d digest.Digest) (int64, error) {
 		if s, ok := sizeCache[d]; ok {

daemon/containerd/snapshotters.go

@@ -0,0 +1,19 @@
+package containerd
+
+import "github.com/containerd/containerd"
+
+// SnapshotterFromGraphDriver returns the containerd snapshotter name based on
+// the supplied graphdriver name. It handles both legacy names and translates
+// them into corresponding containerd snapshotter names.
+func SnapshotterFromGraphDriver(graphDriver string) string {
+	switch graphDriver {
+	case "overlay", "overlay2":
+		return "overlayfs"
+	case "windowsfilter":
+		return "windows"
+	case "":
+		return containerd.DefaultSnapshotter
+	default:
+		return graphDriver
+	}
+}

daemon/containerd/snapshotters_test.go

@@ -0,0 +1,58 @@
+package containerd
+
+import (
+	"testing"
+
+	"github.com/containerd/containerd"
+	"gotest.tools/v3/assert"
+)
+
+func TestSnapshotterFromGraphDriver(t *testing.T) {
+	testCases := []struct {
+		desc     string
+		input    string
+		expected string
+	}{
+		{
+			desc:     "empty defaults to containerd default",
+			input:    "",
+			expected: containerd.DefaultSnapshotter,
+		},
+		{
+			desc:     "overlay -> overlayfs",
+			input:    "overlay",
+			expected: "overlayfs",
+		},
+		{
+			desc:     "overlay2 -> overlayfs",
+			input:    "overlay2",
+			expected: "overlayfs",
+		},
+		{
+			desc:     "windowsfilter -> windows",
+			input:    "windowsfilter",
+			expected: "windows",
+		},
+		{
+			desc:     "containerd overlayfs",
+			input:    "overlayfs",
+			expected: "overlayfs",
+		},
+		{
+			desc:     "containerd zfs",
+			input:    "zfs",
+			expected: "zfs",
+		},
+		{
+			desc:     "unknown is unchanged",
+			input:    "somefuturesnapshotter",
+			expected: "somefuturesnapshotter",
+		},
+	}
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.desc, func(t *testing.T) {
+			assert.Equal(t, SnapshotterFromGraphDriver(tc.input), tc.expected)
+		})
+	}
+}

daemon/create.go

@@ -189,7 +189,7 @@ func (daemon *Daemon) create(ctx context.Context, opts createOpts) (retC *contai
 			return nil, err
 		}
 		parent := identity.ChainID(diffIDs).String()
-		s := daemon.containerdCli.SnapshotService(containerd.DefaultSnapshotter)
+		s := daemon.containerdCli.SnapshotService(daemon.graphDriver)
 		if _, err := s.Prepare(ctx, ctr.ID, parent); err != nil {
 			return nil, err
 		}

daemon/daemon.go

@@ -838,21 +838,6 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		}
 	}
 
-	if isWindows {
-		// On Windows we don't support the environment variable, or a user supplied graphdriver
-		d.graphDriver = "windowsfilter"
-	} else {
-		// Unix platforms however run a single graphdriver for all containers, and it can
-		// be set through an environment variable, a daemon start parameter, or chosen through
-		// initialization of the layerstore through driver priority order for example.
-		if drv := os.Getenv("DOCKER_DRIVER"); drv != "" {
-			d.graphDriver = drv
-			logrus.Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", drv)
-		} else {
-			d.graphDriver = config.GraphDriver // May still be empty. Layerstore init determines instead.
-		}
-	}
-
 	d.registryService = registryService
 	logger.RegisterPluginGetter(d.PluginStore)
 
@@ -942,30 +927,6 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		return nil, err
 	}
 
-	layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{
-		Root:                      config.Root,
-		MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),
-		GraphDriver:               d.graphDriver,
-		GraphDriverOptions:        config.GraphOptions,
-		IDMapping:                 idMapping,
-		PluginGetter:              d.PluginStore,
-		ExperimentalEnabled:       config.Experimental,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	// As layerstore initialization may set the driver
-	d.graphDriver = layerStore.DriverName()
-
-	// Configure and validate the kernels security support. Note this is a Linux/FreeBSD
-	// operation only, so it is safe to pass *just* the runtime OS graphdriver.
-	if err := configureKernelSecuritySupport(config, d.graphDriver); err != nil {
-		return nil, err
-	}
-
-	imageRoot := filepath.Join(config.Root, "image", d.graphDriver)
-
 	d.volumes, err = volumesservice.NewVolumeService(config.Root, d.PluginStore, rootIDs, d)
 	if err != nil {
 		return nil, err
@@ -979,23 +940,6 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 		logrus.WithError(err).Warnf("unable to migrate engine ID; a new engine ID will be generated")
 	}
 
-	// We have a single tag/reference store for the daemon globally. However, it's
-	// stored under the graphdriver. On host platforms which only support a single
-	// container OS, but multiple selectable graphdrivers, this means depending on which
-	// graphdriver is chosen, the global reference store is under there. For
-	// platforms which support multiple container operating systems, this is slightly
-	// more problematic as where does the global ref store get located? Fortunately,
-	// for Windows, which is currently the only daemon supporting multiple container
-	// operating systems, the list of graphdrivers available isn't user configurable.
-	// For backwards compatibility, we just put it under the windowsfilter
-	// directory regardless.
-	refStoreLocation := filepath.Join(imageRoot, `repositories.json`)
-	rs, err := refstore.NewReferenceStore(refStoreLocation)
-	if err != nil {
-		return nil, fmt.Errorf("Couldn't create reference store repository: %s", err)
-	}
-	d.ReferenceStore = rs
-
 	// Check if Devices cgroup is mounted, it is hard requirement for container security,
 	// on Linux.
 	//
@@ -1026,14 +970,74 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S
 
 	d.linkIndex = newLinkIndex()
 
+	// On Windows we don't support the environment variable, or a user supplied graphdriver
+	// Unix platforms however run a single graphdriver for all containers, and it can
+	// be set through an environment variable, a daemon start parameter, or chosen through
+	// initialization of the layerstore through driver priority order for example.
+	graphDriver := os.Getenv("DOCKER_DRIVER")
+	if isWindows {
+		graphDriver = "windowsfilter"
+	} else if graphDriver != "" {
+		logrus.Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", graphDriver)
+	} else {
+		graphDriver = config.GraphDriver
+	}
+
 	if d.UsesSnapshotter() {
-		d.imageService = ctrd.NewService(d.containerdCli, d.containers)
+		snapshotter := ctrd.SnapshotterFromGraphDriver(graphDriver)
+		// Configure and validate the kernels security support. Note this is a Linux/FreeBSD
+		// operation only, so it is safe to pass *just* the runtime OS graphdriver.
+		if err := configureKernelSecuritySupport(config, snapshotter); err != nil {
+			return nil, err
+		}
+		d.imageService = ctrd.NewService(d.containerdCli, d.containers, snapshotter)
+		d.graphDriver = snapshotter
 	} else {
+		layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{
+			Root:                      config.Root,
+			MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"),
+			GraphDriver:               graphDriver,
+			GraphDriverOptions:        config.GraphOptions,
+			IDMapping:                 idMapping,
+			PluginGetter:              d.PluginStore,
+			ExperimentalEnabled:       config.Experimental,
+		})
+		if err != nil {
+			return nil, err
+		}
+
+		// As layerstore initialization may set the driver
+		d.graphDriver = layerStore.DriverName()
+
+		// Configure and validate the kernels security support. Note this is a Linux/FreeBSD
+		// operation only, so it is safe to pass *just* the runtime OS graphdriver.
+		if err := configureKernelSecuritySupport(config, d.graphDriver); err != nil {
+			return nil, err
+		}
+
+		imageRoot := filepath.Join(config.Root, "image", d.graphDriver)
 		ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb"))
 		if err != nil {
 			return nil, err
 		}
 
+		// We have a single tag/reference store for the daemon globally. However, it's
+		// stored under the graphdriver. On host platforms which only support a single
+		// container OS, but multiple selectable graphdrivers, this means depending on which
+		// graphdriver is chosen, the global reference store is under there. For
+		// platforms which support multiple container operating systems, this is slightly
+		// more problematic as where does the global ref store get located? Fortunately,
+		// for Windows, which is currently the only daemon supporting multiple container
+		// operating systems, the list of graphdrivers available isn't user configurable.
+		// For backwards compatibility, we just put it under the windowsfilter
+		// directory regardless.
+		refStoreLocation := filepath.Join(imageRoot, `repositories.json`)
+		rs, err := refstore.NewReferenceStore(refStoreLocation)
+		if err != nil {
+			return nil, fmt.Errorf("Couldn't create reference store repository: %s", err)
+		}
+		d.ReferenceStore = rs
+
 		imageStore, err := image.NewImageStore(ifs, layerStore)
 		if err != nil {
 			return nil, err

daemon/daemon_unix.go

@@ -820,7 +820,7 @@ func configureKernelSecuritySupport(config *config.Config, driverName string) er
 			return nil
 		}
 
-		if driverName == "overlay" || driverName == "overlay2" {
+		if driverName == "overlay" || driverName == "overlay2" || driverName == "overlayfs" {
 			// If driver is overlay or overlay2, make sure kernel
 			// supports selinux with overlay.
 			supported, err := overlaySupportsSelinux()

daemon/oci_linux.go

@@ -11,7 +11,6 @@ import (
 	"strings"
 
 	cdcgroups "github.com/containerd/cgroups"
-	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/containers"
 	coci "github.com/containerd/containerd/oci"
 	"github.com/containerd/containerd/pkg/apparmor"
@@ -1061,10 +1060,19 @@ func (daemon *Daemon) createSpec(ctx context.Context, c *container.Container) (r
 	if daemon.configStore.Rootless {
 		opts = append(opts, WithRootless(daemon))
 	}
+
+	snapshotter := ""
+	snapshotKey := ""
+	if daemon.UsesSnapshotter() {
+		snapshotter = daemon.graphDriver
+		snapshotKey = c.ID
+
+	}
+
 	return &s, coci.ApplyOpts(context.Background(), nil, &containers.Container{
 		ID:          c.ID,
-		Snapshotter: containerd.DefaultSnapshotter,
-		SnapshotKey: c.ID,
+		Snapshotter: snapshotter,
+		SnapshotKey: snapshotKey,
 	}, &s, opts...)
 }