Support restarting containerd in tests, add restart test case

test/cri-containerd/containerdrestart_test.go

@@ -0,0 +1,69 @@
+// +build functional
+
+package cri_containerd
+
+import (
+	"context"
+	"testing"
+
+	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
+)
+
+// CRI will terminate any running containers when it is restarted.
+// Run a container, restart containerd, validate the container is terminated.
+func Test_ContainerdRestart_LCOW(t *testing.T) {
+	requireFeatures(t, featureLCOW, featureTerminateOnRestart)
+
+	pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine})
+
+	client := newTestRuntimeClient(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	sandboxRequest := getRunPodSandboxRequest(t, lcowRuntimeHandler)
+
+	podID := runPodSandbox(t, client, ctx, sandboxRequest)
+	defer removePodSandbox(t, client, ctx, podID)
+	defer stopPodSandbox(t, client, ctx, podID)
+
+	request := &runtime.CreateContainerRequest{
+		PodSandboxId: podID,
+		Config: &runtime.ContainerConfig{
+			Metadata: &runtime.ContainerMetadata{
+				Name: t.Name() + "-Container",
+			},
+			Image: &runtime.ImageSpec{
+				Image: imageLcowAlpine,
+			},
+			Command: []string{
+				"top",
+			},
+		},
+		SandboxConfig: sandboxRequest.Config,
+	}
+
+	containerID := createContainer(t, client, ctx, request)
+	defer removeContainer(t, client, ctx, containerID)
+	startContainer(t, client, ctx, containerID)
+	defer stopContainer(t, client, ctx, containerID)
+
+	t.Log("Restart containerd")
+	stopContainerd(t)
+	startContainerd(t)
+	client = newTestRuntimeClient(t)
+
+	containerStatus, err := client.ContainerStatus(ctx, &runtime.ContainerStatusRequest{ContainerId: containerID})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if containerStatus.Status.State != runtime.ContainerState_CONTAINER_EXITED {
+		t.Errorf("Container was not terminated on containerd restart. Status is %d", containerStatus.Status.State)
+	}
+	podStatus, err := client.PodSandboxStatus(ctx, &runtime.PodSandboxStatusRequest{PodSandboxId: podID})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if podStatus.Status.State != runtime.PodSandboxState_SANDBOX_NOTREADY {
+		t.Errorf("Pod was not terminated on containerd restart. Status is %d", podStatus.Status.State)
+	}
+}

test/cri-containerd/main.go → test/cri-containerd/main_test.go

@@ -78,10 +78,11 @@ var (
 
 // Flags
 var (
-	flagFeatures        = testutilities.NewStringSetFlag()
-	flagCRIEndpoint     = flag.String("cri-endpoint", "tcp://127.0.0.1:2376", "Address of CRI runtime and image service.")
-	flagVirtstack       = flag.String("virtstack", "", "Virtstack to use for hypervisor isolated containers")
-	flagVMServiceBinary = flag.String("vmservice-binary", "", "Path to a binary implementing the vmservice ttrpc service")
+	flagFeatures              = testutilities.NewStringSetFlag()
+	flagCRIEndpoint           = flag.String("cri-endpoint", "tcp://127.0.0.1:2376", "Address of CRI runtime and image service.")
+	flagVirtstack             = flag.String("virtstack", "", "Virtstack to use for hypervisor isolated containers")
+	flagVMServiceBinary       = flag.String("vmservice-binary", "", "Path to a binary implementing the vmservice ttrpc service")
+	flagContainerdServiceName = flag.String("containerd-service-name", "containerd", "Name of the containerd Windows service")
 )
 
 // Features
@@ -94,6 +95,7 @@ const (
 	featureGMSA               = "GMSA"
 	featureGPU                = "GPU"
 	featureCRIUpdateContainer = "UpdateContainer"
+	featureTerminateOnRestart = "TerminateOnRestart"
 )
 
 var allFeatures = []string{
@@ -104,6 +106,7 @@ var allFeatures = []string{
 	featureGMSA,
 	featureGPU,
 	featureCRIUpdateContainer,
+	featureTerminateOnRestart,
 }
 
 func init() {

test/cri-containerd/service_test.go

@@ -0,0 +1,90 @@
+package cri_containerd
+
+import (
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/sys/windows/svc"
+	"golang.org/x/sys/windows/svc/mgr"
+)
+
+// Implements functionality so tests can start/stop the containerd service.
+// Tests assume containerd will be running when they start, since this
+// matches with the state a dev box will usually be in. A test that stops containerd should
+// therefore ensure it starts it again.
+
+var (
+	svcMgr            *mgr.Mgr
+	svcMgrConnectOnce sync.Once
+	svcMgrConnectErr  error
+)
+
+func getSvcMgr() (*mgr.Mgr, error) {
+	svcMgrConnectOnce.Do(func() {
+		s, err := mgr.Connect()
+		if err != nil {
+			err = fmt.Errorf("failed to connect to service manager: %w", err)
+		}
+		svcMgr, svcMgrConnectErr = s, err
+	})
+	return svcMgr, svcMgrConnectErr
+}
+
+func startService(serviceName string) error {
+	m, err := getSvcMgr()
+	if err != nil {
+		return err
+	}
+	s, err := m.OpenService(serviceName)
+	if err != nil {
+		return fmt.Errorf("failed to open service %s: %w", serviceName, err)
+	}
+	if err := s.Start(); err != nil {
+		return fmt.Errorf("failed to start service %s: %w", serviceName, err)
+	}
+	return nil
+}
+
+func stopService(serviceName string) error {
+	m, err := getSvcMgr()
+	if err != nil {
+		return err
+	}
+	s, err := m.OpenService(serviceName)
+	if err != nil {
+		return fmt.Errorf("failed to open service %s: %w", serviceName, err)
+	}
+	status, err := s.Control(svc.Stop)
+	if err != nil {
+		return fmt.Errorf("failed to send stop control to service %s: %w", serviceName, err)
+	}
+	tc := time.NewTimer(10 * time.Second)
+	defer tc.Stop()
+	for status.State != svc.Stopped {
+		time.Sleep(1 * time.Second)
+		select {
+		case <-tc.C:
+			return fmt.Errorf("service %s did not stop in time", serviceName)
+		default:
+			status, err = s.Query()
+			if err != nil {
+				return fmt.Errorf("failed to query service %s status: %w", serviceName, err)
+			}
+		}
+	}
+	return nil
+}
+
+func startContainerd(t *testing.T) {
+	if err := startService(*flagContainerdServiceName); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func stopContainerd(t *testing.T) {
+	if err := stopService(*flagContainerdServiceName); err != nil {
+		t.Fatal(err)
+	}
+}