Implement early cloning for UVM

cmd/containerd-shim-runhcs-v1/pod.go

@@ -51,53 +51,62 @@ type shimPod interface {
 	// the `shimExecStateRunning, shimExecStateExited` states. If the exec is
 	// not in this state this pod MUST return `errdefs.ErrFailedPrecondition`.
 	KillTask(ctx context.Context, tid, eid string, signal uint32, all bool) error
+
+	// Specifies if this pod is a template
+	IsTemplate() bool
 }
 
-func createPod(ctx context.Context, events publisher, req *task.CreateTaskRequest, s *specs.Spec) (shimPod, error) {
+// TODO (ambarve): The third return value of this function (the template id)
+// is only needed until we implement the late cloning part after that this return
+// value can be removed.
+func createPod(ctx context.Context, events publisher, req *task.CreateTaskRequest, s *specs.Spec) (shimPod, error, string) {
 	log.G(ctx).WithField("tid", req.ID).Debug("createPod")
 
 	if osversion.Get().Build < osversion.RS5 {
-		return nil, errors.Wrapf(errdefs.ErrFailedPrecondition, "pod support is not available on Windows versions previous to RS5 (%d)", osversion.RS5)
+		return nil, errors.Wrapf(errdefs.ErrFailedPrecondition, "pod support is not available on Windows versions previous to RS5 (%d)", osversion.RS5), ""
 	}
 
 	ct, sid, err := oci.GetSandboxTypeAndID(s.Annotations)
 	if err != nil {
-		return nil, err
+		return nil, err, ""
 	}
 	if ct != oci.KubernetesContainerTypeSandbox {
 		return nil, errors.Wrapf(
 			errdefs.ErrFailedPrecondition,
 			"expected annotation: '%s': '%s' got '%s'",
 			oci.KubernetesContainerTypeAnnotation,
 			oci.KubernetesContainerTypeSandbox,
-			ct)
+			ct), ""
 	}
 	if sid != req.ID {
 		return nil, errors.Wrapf(
 			errdefs.ErrFailedPrecondition,
 			"expected annotation '%s': '%s' got '%s'",
 			oci.KubernetesSandboxIDAnnotation,
 			req.ID,
-			sid)
+			sid), ""
 	}
 
 	owner := filepath.Base(os.Args[0])
 	isWCOW := oci.IsWCOW(s)
 
 	var parent *uvm.UtilityVM
+	var isCreateTemplateRequest bool
 	if oci.IsIsolated(s) {
 		// Create the UVM parent
 		opts, err := oci.SpecToUVMCreateOpts(ctx, s, fmt.Sprintf("%s@vm", req.ID), owner)
 		if err != nil {
-			return nil, err
+			return nil, err, ""
 		}
 		switch opts.(type) {
 		case *uvm.OptionsLCOW:
 			lopts := (opts).(*uvm.OptionsLCOW)
 			parent, err = uvm.CreateLCOW(ctx, lopts)
 			if err != nil {
-				return nil, err
+				return nil, err, ""
 			}
+			// TODO(ambarve): implement cloning for LCOW
+			isCreateTemplateRequest = lopts.Options.SaveAsTemplate
 		case *uvm.OptionsWCOW:
 			wopts := (opts).(*uvm.OptionsWCOW)
 
@@ -111,23 +120,24 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
 			vmPath := filepath.Join(layers[layersLen-1], "vm")
 			err := os.MkdirAll(vmPath, 0)
 			if err != nil {
-				return nil, err
+				return nil, err, ""
 			}
 			layers[layersLen-1] = vmPath
 			wopts.LayerFolders = layers
 
 			parent, err = uvm.CreateWCOW(ctx, wopts)
 			if err != nil {
-				return nil, err
+				return nil, err, ""
 			}
+			isCreateTemplateRequest = wopts.Options.SaveAsTemplate
 		}
 		err = parent.Start(ctx)
 		if err != nil {
 			parent.Close()
-			return nil, err
+			return nil, err, ""
 		}
 	} else if !isWCOW {
-		return nil, errors.Wrap(errdefs.ErrFailedPrecondition, "oci spec does not contain WCOW or LCOW spec")
+		return nil, errors.Wrap(errdefs.ErrFailedPrecondition, "oci spec does not contain WCOW or LCOW spec"), ""
 	}
 	defer func() {
 		// clean up the uvm if we fail any further operations
@@ -137,10 +147,22 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
 	}()
 
 	p := pod{
-		events: events,
-		id:     req.ID,
-		host:   parent,
+		events:     events,
+		id:         req.ID,
+		host:       parent,
+		isTemplate: isCreateTemplateRequest,
+	}
+
+	// For a template creation request return before actually starting
+	// any containers inside it.
+	if isCreateTemplateRequest {
+		err = parent.SaveAsTemplate(ctx)
+		if err != nil {
+			return nil, err, ""
+		}
+		return &p, nil, parent.ID()
 	}
+
 	// TOOD: JTERRY75 - There is a bug in the compartment activation for Windows
 	// Process isolated that requires us to create the real pause container to
 	// hold the network compartment open. This is not required for Windows
@@ -160,15 +182,15 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
 			if nsid != "" {
 				endpoints, err := hcsoci.GetNamespaceEndpoints(ctx, nsid)
 				if err != nil {
-					return nil, err
+					return nil, err, ""
 				}
 				err = parent.AddNetNS(ctx, nsid)
 				if err != nil {
-					return nil, err
+					return nil, err, ""
 				}
 				err = parent.AddEndpointsToNS(ctx, nsid, endpoints)
 				if err != nil {
-					return nil, err
+					return nil, err, ""
 				}
 			}
 		}
@@ -202,12 +224,12 @@ func createPod(ctx context.Context, events publisher, req *task.CreateTaskReques
 		// task for the sandbox.
 		lt, err := newHcsTask(ctx, events, parent, true, req, s)
 		if err != nil {
-			return nil, err
+			return nil, err, ""
 		}
 		p.sandboxTask = lt
 	}
 
-	return &p, nil
+	return &p, nil, ""
 }
 
 var _ = (shimPod)(&pod{})
@@ -235,6 +257,9 @@ type pod struct {
 	// to release the lock to allow concurrent creates.
 	wcl           sync.Mutex
 	workloadTasks sync.Map
+
+	// specifies if this pod was created as a template.
+	isTemplate bool
 }
 
 func (p *pod) ID() string {
@@ -329,3 +354,7 @@ func (p *pod) KillTask(ctx context.Context, tid, eid string, signal uint32, all
 	})
 	return eg.Wait()
 }
+
+func (p *pod) IsTemplate() bool {
+	return p.isTemplate
+}

cmd/containerd-shim-runhcs-v1/service_internal.go

@@ -79,7 +79,6 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
 		}
 		shimOpts = v.(*runhcsopts.Options)
 	}
-
 	var spec specs.Spec
 	f, err := os.Open(filepath.Join(req.Bundle, "config.json"))
 	if err != nil {
@@ -163,15 +162,19 @@ func (s *service) createInternal(ctx context.Context, req *task.CreateTaskReques
 			resp.Pid = uint32(e.Pid())
 			return resp, nil
 		}
-		pod, err = createPod(ctx, s.events, req, &spec)
+		pod, err, templateID := createPod(ctx, s.events, req, &spec)
 		if err != nil {
 			s.cl.Unlock()
 			return nil, err
 		}
-		t, _ := pod.GetTask(req.ID)
-		e, _ := t.GetExec("")
-		resp.Pid = uint32(e.Pid())
-		s.taskOrPod.Store(pod)
+		if templateID == "" {
+			t, _ := pod.GetTask(req.ID)
+			e, _ := t.GetExec("")
+			resp.Pid = uint32(e.Pid())
+			s.taskOrPod.Store(pod)
+		}
+		// TODO(ambarve): A template pod probably should still be
+		// stored in the taskOrPod map until late cloning is implemented
 	} else {
 		t, err := newHcsStandaloneTask(ctx, s.events, req, &spec)
 		if err != nil {

internal/hcs/callback.go

@@ -106,6 +106,7 @@ func newSystemChannels() notificationChannels {
 		hcsNotificationSystemStartCompleted,
 		hcsNotificationSystemPauseCompleted,
 		hcsNotificationSystemResumeCompleted,
+		hcsNotificationSystemSaveCompleted,
 	} {
 		channels[notif] = make(notificationChannel, 1)
 	}

internal/hcs/system.go

@@ -604,3 +604,30 @@ func (computeSystem *System) Modify(ctx context.Context, config interface{}) err
 
 	return nil
 }
+
+// Save the compute system
+func (computeSystem *System) Save(ctx context.Context, options string) (err error) {
+	operation := "hcsshim::System::Save"
+
+	// hcsSaveComputeSystemContext is an async peration. Start the outer span
+	// here to measure the full save time.
+	ctx, span := trace.StartSpan(ctx, operation)
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(trace.StringAttribute("cid", computeSystem.id))
+
+	computeSystem.handleLock.RLock()
+	defer computeSystem.handleLock.RUnlock()
+
+	if computeSystem.handle == 0 {
+		return makeSystemError(computeSystem, operation, "", ErrAlreadyClosed, nil)
+	}
+
+	result, err := vmcompute.HcsSaveComputeSystem(ctx, computeSystem.handle, options)
+	events, err := processAsyncHcsResult(ctx, err, result, computeSystem.callbackNumber, hcsNotificationSystemSaveCompleted, &timeout.SystemSave)
+	if err != nil {
+		return makeSystemError(computeSystem, operation, "", err, events)
+	}
+
+	return nil
+}

internal/hcsoci/resources_wcow.go

@@ -19,6 +19,8 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
+const wcowGlobalMountPrefix = "C:\\mounts\\m%d"
+
 func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, resources *Resources) error {
 	if coi.Spec == nil || coi.Spec.Windows == nil || coi.Spec.Windows.LayerFolders == nil {
 		return fmt.Errorf("field 'Spec.Windows.Layerfolders' is not populated")
@@ -77,8 +79,7 @@ func allocateWindowsResources(ctx context.Context, coi *createOptionsInternal, r
 		}
 
 		if coi.HostingSystem != nil && schemaversion.IsV21(coi.actualSchemaVersion) {
-			uvmPath := fmt.Sprintf("C:\\%s\\%d", coi.actualID, i)
-
+			uvmPath := fmt.Sprintf(wcowGlobalMountPrefix, coi.HostingSystem.UVMMountCounter())
 			readOnly := false
 			for _, o := range mount.Options {
 				if strings.ToLower(o) == "ro" {

internal/oci/uvm.go

@@ -123,6 +123,8 @@ const (
 	annotationVPCIEnabled                = "io.microsoft.virtualmachine.lcow.vpcienabled"
 	annotationStorageQoSBandwidthMaximum = "io.microsoft.virtualmachine.storageqos.bandwidthmaximum"
 	annotationStorageQoSIopsMaximum      = "io.microsoft.virtualmachine.storageqos.iopsmaximum"
+	annotationSaveAsTemplate             = "io.microsoft.virtualmachine.saveastemplate"
+	annotationTemplateID                 = "io.microsoft.virtualmachine.templateid"
 )
 
 // parseAnnotationsBool searches `a` for `key` and if found verifies that the
@@ -348,6 +350,8 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) (
 			lopts.KernelFile = uvm.KernelFile
 		}
 		lopts.BootFilesPath = parseAnnotationsString(s.Annotations, annotationBootFilesRootPath, lopts.BootFilesPath)
+		lopts.SaveAsTemplate = parseAnnotationsBool(ctx, s.Annotations, annotationSaveAsTemplate, false)
+		lopts.TemplateID = parseAnnotationsString(s.Annotations, annotationTemplateID, lopts.TemplateID)
 		return lopts, nil
 	} else if IsWCOW(s) {
 		wopts := uvm.NewDefaultOptionsWCOW(id, owner)
@@ -362,6 +366,8 @@ func SpecToUVMCreateOpts(ctx context.Context, s *specs.Spec, id, owner string) (
 		wopts.ProcessorWeight = ParseAnnotationsCPUWeight(ctx, s, annotationProcessorWeight, wopts.ProcessorWeight)
 		wopts.StorageQoSBandwidthMaximum = ParseAnnotationsStorageBps(ctx, s, annotationStorageQoSBandwidthMaximum, wopts.StorageQoSBandwidthMaximum)
 		wopts.StorageQoSIopsMaximum = ParseAnnotationsStorageIops(ctx, s, annotationStorageQoSIopsMaximum, wopts.StorageQoSIopsMaximum)
+		wopts.SaveAsTemplate = parseAnnotationsBool(ctx, s.Annotations, annotationSaveAsTemplate, false)
+		wopts.TemplateID = parseAnnotationsString(s.Annotations, annotationTemplateID, wopts.TemplateID)
 		return wopts, nil
 	}
 	return nil, errors.New("cannot create UVM opts spec is not LCOW or WCOW")

internal/timeout/timeout.go

@@ -29,6 +29,9 @@ var (
 	// SystemResume is the timeout for resuming a compute system
 	SystemResume time.Duration = defaultTimeout
 
+	// SystemSave is the timeout for saving a compute system
+	SystemSave time.Duration = defaultTimeout
+
 	// SyscallWatcher is the timeout before warning of a potential stuck platform syscall.
 	SyscallWatcher time.Duration = defaultTimeout
 
@@ -51,6 +54,7 @@ func init() {
 	SystemStart = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMSTART", SystemStart)
 	SystemPause = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMPAUSE", SystemPause)
 	SystemResume = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMRESUME", SystemResume)
+	SystemSave = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSTEMSAVE", SystemSave)
 	SyscallWatcher = durationFromEnvironment("HCSSHIM_TIMEOUT_SYSCALLWATCHER", SyscallWatcher)
 	Tar2VHD = durationFromEnvironment("HCSSHIM_TIMEOUT_TAR2VHD", Tar2VHD)
 	ExternalCommandToStart = durationFromEnvironment("HCSSHIM_TIMEOUT_EXTERNALCOMMANDSTART", ExternalCommandToStart)