diff --git a/daemon/config/config.go b/daemon/config/config.go index 645beb8c032f6..25fc5091aba1c 100644 --- a/daemon/config/config.go +++ b/daemon/config/config.go @@ -10,6 +10,7 @@ import ( "strings" "sync" + "github.com/containerd/containerd" "github.com/docker/docker/opts" "github.com/docker/docker/pkg/authorization" "github.com/docker/docker/registry" @@ -51,6 +52,9 @@ const ( // DefaultPluginNamespace is the name of the default containerd namespace used for plugins. DefaultPluginNamespace = "plugins.moby" + // DefaultContainerdSnapshotter is the name of the default containerd snapshotter used for creating container root fs + DefaultContainerdSnapshotter = containerd.DefaultSnapshotter + // LinuxV2RuntimeName is the runtime used to specify the containerd v2 runc shim LinuxV2RuntimeName = "io.containerd.runc.v2" diff --git a/daemon/containerd/image_exporter.go b/daemon/containerd/image_exporter.go index 758f57c2af149..679f3b1c2ec24 100644 --- a/daemon/containerd/image_exporter.go +++ b/daemon/containerd/image_exporter.go @@ -31,14 +31,14 @@ func (i *ImageService) LoadImage(ctx context.Context, inTar io.ReadCloser, outSt for _, img := range imgs { platformImg := containerd.NewImageWithPlatform(i.client, img, platform) - unpacked, err := platformImg.IsUnpacked(ctx, containerd.DefaultSnapshotter) + unpacked, err := platformImg.IsUnpacked(ctx, i.snapshotter) if err != nil { logrus.WithError(err).WithField("image", img.Name).Error("IsUnpacked failed") continue } if !unpacked { - err := platformImg.Unpack(ctx, containerd.DefaultSnapshotter) + err := platformImg.Unpack(ctx, i.snapshotter) if err != nil { logrus.WithError(err).WithField("image", img.Name).Error("Failed to unpack image") return errors.Wrapf(err, "Failed to unpack image") diff --git a/daemon/containerd/image_list.go b/daemon/containerd/image_list.go index 05c040a3c7caf..0818d34533478 100644 --- a/daemon/containerd/image_list.go +++ b/daemon/containerd/image_list.go @@ -37,7 +37,7 @@ func (i *ImageService) Images(ctx context.Context, opts types.ImageListOptions) return nil, err } - snapshotter := i.client.SnapshotService(containerd.DefaultSnapshotter) + snapshotter := i.client.SnapshotService(i.snapshotter) sizeCache := make(map[digest.Digest]int64) snapshotSizeFn := func(d digest.Digest) (int64, error) { if s, ok := sizeCache[d]; ok { diff --git a/daemon/containerd/image_pull.go b/daemon/containerd/image_pull.go index bcc6e9ff7e48b..dff6bd0a29b0b 100644 --- a/daemon/containerd/image_pull.go +++ b/daemon/containerd/image_pull.go @@ -62,13 +62,13 @@ func (i *ImageService) PullImage(ctx context.Context, image, tagOrDigest string, return err } - unpacked, err := img.IsUnpacked(ctx, containerd.DefaultSnapshotter) + unpacked, err := img.IsUnpacked(ctx, i.snapshotter) if err != nil { return err } if !unpacked { - if err := img.Unpack(ctx, containerd.DefaultSnapshotter); err != nil { + if err := img.Unpack(ctx, i.snapshotter); err != nil { return err } } diff --git a/daemon/containerd/service.go b/daemon/containerd/service.go index a2a80e78bc88f..563e9a00b29e8 100644 --- a/daemon/containerd/service.go +++ b/daemon/containerd/service.go @@ -20,16 +20,18 @@ import ( // ImageService implements daemon.ImageService type ImageService struct { - client *containerd.Client - usage singleflight.Group - containers container.Store + client *containerd.Client + usage singleflight.Group + containers container.Store + snapshotter string } // NewService creates a new ImageService. -func NewService(c *containerd.Client, containers container.Store) *ImageService { +func NewService(c *containerd.Client, containers container.Store, snapshotter string) *ImageService { return &ImageService{ - client: c, - containers: containers, + client: c, + containers: containers, + snapshotter: snapshotter, } } @@ -93,7 +95,7 @@ func (i *ImageService) Cleanup() error { // - newContainer // - to report an error in Daemon.Mount(container) func (i *ImageService) GraphDriverName() string { - return "containerd-snapshotter" + return i.snapshotter } // ReleaseLayer releases a layer allowing it to be removed @@ -107,7 +109,7 @@ func (i *ImageService) ReleaseLayer(rwlayer layer.RWLayer) error { func (i *ImageService) LayerDiskUsage(ctx context.Context) (int64, error) { ch := i.usage.DoChan("LayerDiskUsage", func() (interface{}, error) { var allLayersSize int64 - snapshotter := i.client.SnapshotService(containerd.DefaultSnapshotter) + snapshotter := i.client.SnapshotService(i.snapshotter) snapshotter.Walk(ctx, func(ctx context.Context, info snapshots.Info) error { usage, err := snapshotter.Usage(ctx, info.Name) if err != nil { @@ -168,7 +170,7 @@ func (i *ImageService) GetLayerFolders(img *image.Image, rwLayer layer.RWLayer) // GetContainerLayerSize returns the real size & virtual size of the container. func (i *ImageService) GetContainerLayerSize(ctx context.Context, containerID string) (int64, int64, error) { - snapshotter := i.client.SnapshotService(containerd.DefaultSnapshotter) + snapshotter := i.client.SnapshotService(i.snapshotter) sizeCache := make(map[digest.Digest]int64) snapshotSizeFn := func(d digest.Digest) (int64, error) { if s, ok := sizeCache[d]; ok { diff --git a/daemon/containerd/snapshotters.go b/daemon/containerd/snapshotters.go new file mode 100644 index 0000000000000..bd9e4766827a7 --- /dev/null +++ b/daemon/containerd/snapshotters.go @@ -0,0 +1,19 @@ +package containerd + +import "github.com/containerd/containerd" + +// SnapshotterFromGraphDriver returns the containerd snapshotter name based on +// the supplied graphdriver name. It handles both legacy names and translates +// them into corresponding containerd snapshotter names. +func SnapshotterFromGraphDriver(graphDriver string) string { + switch graphDriver { + case "overlay", "overlay2": + return "overlayfs" + case "windowsfilter": + return "windows" + case "": + return containerd.DefaultSnapshotter + default: + return graphDriver + } +} diff --git a/daemon/containerd/snapshotters_test.go b/daemon/containerd/snapshotters_test.go new file mode 100644 index 0000000000000..9d05794bb5cfa --- /dev/null +++ b/daemon/containerd/snapshotters_test.go @@ -0,0 +1,58 @@ +package containerd + +import ( + "testing" + + "github.com/containerd/containerd" + "gotest.tools/v3/assert" +) + +func TestSnapshotterFromGraphDriver(t *testing.T) { + testCases := []struct { + desc string + input string + expected string + }{ + { + desc: "empty defaults to containerd default", + input: "", + expected: containerd.DefaultSnapshotter, + }, + { + desc: "overlay -> overlayfs", + input: "overlay", + expected: "overlayfs", + }, + { + desc: "overlay2 -> overlayfs", + input: "overlay2", + expected: "overlayfs", + }, + { + desc: "windowsfilter -> windows", + input: "windowsfilter", + expected: "windows", + }, + { + desc: "containerd overlayfs", + input: "overlayfs", + expected: "overlayfs", + }, + { + desc: "containerd zfs", + input: "zfs", + expected: "zfs", + }, + { + desc: "unknown is unchanged", + input: "somefuturesnapshotter", + expected: "somefuturesnapshotter", + }, + } + for _, tc := range testCases { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + assert.Equal(t, SnapshotterFromGraphDriver(tc.input), tc.expected) + }) + } +} diff --git a/daemon/create.go b/daemon/create.go index 5b2ec16c0d6f1..f75bd0bc124c7 100644 --- a/daemon/create.go +++ b/daemon/create.go @@ -189,7 +189,7 @@ func (daemon *Daemon) create(ctx context.Context, opts createOpts) (retC *contai return nil, err } parent := identity.ChainID(diffIDs).String() - s := daemon.containerdCli.SnapshotService(containerd.DefaultSnapshotter) + s := daemon.containerdCli.SnapshotService(daemon.graphDriver) if _, err := s.Prepare(ctx, ctr.ID, parent); err != nil { return nil, err } diff --git a/daemon/daemon.go b/daemon/daemon.go index 45e2f0f675d38..e59be1edd5b5a 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -838,21 +838,6 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S } } - if isWindows { - // On Windows we don't support the environment variable, or a user supplied graphdriver - d.graphDriver = "windowsfilter" - } else { - // Unix platforms however run a single graphdriver for all containers, and it can - // be set through an environment variable, a daemon start parameter, or chosen through - // initialization of the layerstore through driver priority order for example. - if drv := os.Getenv("DOCKER_DRIVER"); drv != "" { - d.graphDriver = drv - logrus.Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", drv) - } else { - d.graphDriver = config.GraphDriver // May still be empty. Layerstore init determines instead. - } - } - d.registryService = registryService logger.RegisterPluginGetter(d.PluginStore) @@ -942,30 +927,6 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S return nil, err } - layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{ - Root: config.Root, - MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"), - GraphDriver: d.graphDriver, - GraphDriverOptions: config.GraphOptions, - IDMapping: idMapping, - PluginGetter: d.PluginStore, - ExperimentalEnabled: config.Experimental, - }) - if err != nil { - return nil, err - } - - // As layerstore initialization may set the driver - d.graphDriver = layerStore.DriverName() - - // Configure and validate the kernels security support. Note this is a Linux/FreeBSD - // operation only, so it is safe to pass *just* the runtime OS graphdriver. - if err := configureKernelSecuritySupport(config, d.graphDriver); err != nil { - return nil, err - } - - imageRoot := filepath.Join(config.Root, "image", d.graphDriver) - d.volumes, err = volumesservice.NewVolumeService(config.Root, d.PluginStore, rootIDs, d) if err != nil { return nil, err @@ -979,23 +940,6 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S logrus.WithError(err).Warnf("unable to migrate engine ID; a new engine ID will be generated") } - // We have a single tag/reference store for the daemon globally. However, it's - // stored under the graphdriver. On host platforms which only support a single - // container OS, but multiple selectable graphdrivers, this means depending on which - // graphdriver is chosen, the global reference store is under there. For - // platforms which support multiple container operating systems, this is slightly - // more problematic as where does the global ref store get located? Fortunately, - // for Windows, which is currently the only daemon supporting multiple container - // operating systems, the list of graphdrivers available isn't user configurable. - // For backwards compatibility, we just put it under the windowsfilter - // directory regardless. - refStoreLocation := filepath.Join(imageRoot, `repositories.json`) - rs, err := refstore.NewReferenceStore(refStoreLocation) - if err != nil { - return nil, fmt.Errorf("Couldn't create reference store repository: %s", err) - } - d.ReferenceStore = rs - // Check if Devices cgroup is mounted, it is hard requirement for container security, // on Linux. // @@ -1026,14 +970,74 @@ func NewDaemon(ctx context.Context, config *config.Config, pluginStore *plugin.S d.linkIndex = newLinkIndex() + // On Windows we don't support the environment variable, or a user supplied graphdriver + // Unix platforms however run a single graphdriver for all containers, and it can + // be set through an environment variable, a daemon start parameter, or chosen through + // initialization of the layerstore through driver priority order for example. + graphDriver := os.Getenv("DOCKER_DRIVER") + if isWindows { + graphDriver = "windowsfilter" + } else if graphDriver != "" { + logrus.Infof("Setting the storage driver from the $DOCKER_DRIVER environment variable (%s)", graphDriver) + } else { + graphDriver = config.GraphDriver + } + if d.UsesSnapshotter() { - d.imageService = ctrd.NewService(d.containerdCli, d.containers) + snapshotter := ctrd.SnapshotterFromGraphDriver(graphDriver) + // Configure and validate the kernels security support. Note this is a Linux/FreeBSD + // operation only, so it is safe to pass *just* the runtime OS graphdriver. + if err := configureKernelSecuritySupport(config, snapshotter); err != nil { + return nil, err + } + d.imageService = ctrd.NewService(d.containerdCli, d.containers, snapshotter) + d.graphDriver = snapshotter } else { + layerStore, err := layer.NewStoreFromOptions(layer.StoreOptions{ + Root: config.Root, + MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"), + GraphDriver: graphDriver, + GraphDriverOptions: config.GraphOptions, + IDMapping: idMapping, + PluginGetter: d.PluginStore, + ExperimentalEnabled: config.Experimental, + }) + if err != nil { + return nil, err + } + + // As layerstore initialization may set the driver + d.graphDriver = layerStore.DriverName() + + // Configure and validate the kernels security support. Note this is a Linux/FreeBSD + // operation only, so it is safe to pass *just* the runtime OS graphdriver. + if err := configureKernelSecuritySupport(config, d.graphDriver); err != nil { + return nil, err + } + + imageRoot := filepath.Join(config.Root, "image", d.graphDriver) ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb")) if err != nil { return nil, err } + // We have a single tag/reference store for the daemon globally. However, it's + // stored under the graphdriver. On host platforms which only support a single + // container OS, but multiple selectable graphdrivers, this means depending on which + // graphdriver is chosen, the global reference store is under there. For + // platforms which support multiple container operating systems, this is slightly + // more problematic as where does the global ref store get located? Fortunately, + // for Windows, which is currently the only daemon supporting multiple container + // operating systems, the list of graphdrivers available isn't user configurable. + // For backwards compatibility, we just put it under the windowsfilter + // directory regardless. + refStoreLocation := filepath.Join(imageRoot, `repositories.json`) + rs, err := refstore.NewReferenceStore(refStoreLocation) + if err != nil { + return nil, fmt.Errorf("Couldn't create reference store repository: %s", err) + } + d.ReferenceStore = rs + imageStore, err := image.NewImageStore(ifs, layerStore) if err != nil { return nil, err diff --git a/daemon/daemon_unix.go b/daemon/daemon_unix.go index e33bc8383d91d..b3c126353d667 100644 --- a/daemon/daemon_unix.go +++ b/daemon/daemon_unix.go @@ -820,7 +820,7 @@ func configureKernelSecuritySupport(config *config.Config, driverName string) er return nil } - if driverName == "overlay" || driverName == "overlay2" { + if driverName == "overlay" || driverName == "overlay2" || driverName == "overlayfs" { // If driver is overlay or overlay2, make sure kernel // supports selinux with overlay. supported, err := overlaySupportsSelinux() diff --git a/daemon/oci_linux.go b/daemon/oci_linux.go index 8a61da2c2cebb..5bdd8dccfcd10 100644 --- a/daemon/oci_linux.go +++ b/daemon/oci_linux.go @@ -11,7 +11,6 @@ import ( "strings" cdcgroups "github.com/containerd/cgroups" - "github.com/containerd/containerd" "github.com/containerd/containerd/containers" coci "github.com/containerd/containerd/oci" "github.com/containerd/containerd/pkg/apparmor" @@ -1061,10 +1060,19 @@ func (daemon *Daemon) createSpec(ctx context.Context, c *container.Container) (r if daemon.configStore.Rootless { opts = append(opts, WithRootless(daemon)) } + + snapshotter := "" + snapshotKey := "" + if daemon.UsesSnapshotter() { + snapshotter = daemon.graphDriver + snapshotKey = c.ID + + } + return &s, coci.ApplyOpts(context.Background(), nil, &containers.Container{ ID: c.ID, - Snapshotter: containerd.DefaultSnapshotter, - SnapshotKey: c.ID, + Snapshotter: snapshotter, + SnapshotKey: snapshotKey, }, &s, opts...) } diff --git a/daemon/start.go b/daemon/start.go index 21385af074d6b..f3318618219b6 100644 --- a/daemon/start.go +++ b/daemon/start.go @@ -180,7 +180,7 @@ func (daemon *Daemon) containerStart(ctx context.Context, container *container.C newContainerOpts := []containerd.NewContainerOpts{} if daemon.UsesSnapshotter() { - newContainerOpts = append(newContainerOpts, containerd.WithSnapshotter(containerd.DefaultSnapshotter)) + newContainerOpts = append(newContainerOpts, containerd.WithSnapshotter(daemon.graphDriver)) newContainerOpts = append(newContainerOpts, containerd.WithSnapshot(container.ID)) c8dImge, err := daemon.imageService.(containerdImage).GetContainerdImage(ctx, container.Config.Image, &v1.Platform{}) if err != nil {