From f9fa0e6f221bba8e581bc1919db14bcc2031dc7b Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Tue, 18 Aug 2020 12:01:24 -0700 Subject: [PATCH] Add end to end tests for late cloning. This is one of the many small PRs that enable the support for late cloning. This commit adds several end to end tests for the late cloning feature. Signed-off-by: Amit Barve --- test/cri-containerd/clone_test.go | 482 ++++++++++++++++++++++++++++++ test/cri-containerd/container.go | 10 + 2 files changed, 492 insertions(+) create mode 100644 test/cri-containerd/clone_test.go diff --git a/test/cri-containerd/clone_test.go b/test/cri-containerd/clone_test.go new file mode 100644 index 0000000000..0c96912cc8 --- /dev/null +++ b/test/cri-containerd/clone_test.go @@ -0,0 +1,482 @@ +// +build functional + +package cri_containerd + +import ( + "context" + "fmt" + "os/exec" + "strings" + "sync" + "testing" + "time" + + runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" +) + +// returns a request config for creating a template sandbox +func getTemplatePodConfig(name string) *runtime.RunPodSandboxRequest { + return &runtime.RunPodSandboxRequest{ + Config: &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: name, + Uid: "0", + Namespace: testNamespace, + }, + Annotations: map[string]string{ + "io.microsoft.virtualmachine.saveastemplate": "true", + }, + }, + RuntimeHandler: wcowHypervisorRuntimeHandler, + } +} + +// returns a request config for creating a template container +func getTemplateContainerConfig(name string) *runtime.CreateContainerRequest { + return &runtime.CreateContainerRequest{ + Config: &runtime.ContainerConfig{ + Metadata: &runtime.ContainerMetadata{ + Name: name, + }, + Image: &runtime.ImageSpec{ + Image: imageWindowsNanoserver, + }, + // Do not keep the ping running on template containers. + Command: []string{ + "ping", + "127.0.0.1", + }, + Annotations: map[string]string{ + "io.microsoft.virtualmachine.saveastemplate": "true", + }, + }, + } +} + +// returns a request config for creating a standard container +func getStandardContainerConfig(name string) *runtime.CreateContainerRequest { + return &runtime.CreateContainerRequest{ + Config: &runtime.ContainerConfig{ + Metadata: &runtime.ContainerMetadata{ + Name: name, + }, + Image: &runtime.ImageSpec{ + Image: imageWindowsNanoserver, + }, + Command: []string{ + "ping", + "-t", + "127.0.0.1", + }, + }, + } +} + +// returns a create cloned sandbox request config. +func getClonedPodConfig(uniqueID int, templateid string) *runtime.RunPodSandboxRequest { + return &runtime.RunPodSandboxRequest{ + Config: &runtime.PodSandboxConfig{ + Metadata: &runtime.PodSandboxMetadata{ + Name: fmt.Sprintf("clonedpod-%d", uniqueID), + Uid: "0", + Namespace: testNamespace, + }, + Annotations: map[string]string{ + "io.microsoft.virtualmachine.templateid": templateid + "@vm", + }, + }, + RuntimeHandler: wcowHypervisorRuntimeHandler, + } +} + +// returns a create cloned container request config. +func getClonedContainerConfig(uniqueID int, templateid string) *runtime.CreateContainerRequest { + return &runtime.CreateContainerRequest{ + Config: &runtime.ContainerConfig{ + Metadata: &runtime.ContainerMetadata{ + Name: fmt.Sprintf("clonedcontainer-%d", uniqueID), + }, + Image: &runtime.ImageSpec{ + Image: imageWindowsNanoserver, + }, + // Command for cloned containers + Command: []string{ + "ping", + "-t", + "127.0.0.1", + }, + Annotations: map[string]string{ + "io.microsoft.virtualmachine.templateid": templateid, + }, + }, + } +} + +func waitForTemplateSave(ctx context.Context, t *testing.T, templatePodID string) { + app := "hcsdiag" + arg0 := "list" + for { + cmd := exec.Command(app, arg0) + stdout, err := cmd.Output() + if err != nil { + t.Fatalf("failed while waiting for save template to finish: %s", err) + } + if strings.Contains(string(stdout), templatePodID) && strings.Contains(string(stdout), "SavedAsTemplate") { + break + } + timer := time.NewTimer(time.Millisecond * 100) + select { + case <-ctx.Done(): + t.Fatalf("Timelimit exceeded for wait for template saving to finish") + case <-timer.C: + } + } +} + +func createPodAndContainer(ctx context.Context, t *testing.T, client runtime.RuntimeServiceClient, sandboxRequest *runtime.RunPodSandboxRequest, containerRequest *runtime.CreateContainerRequest, podID, containerID *string) { + // This is required in order to avoid leaking a pod and/or container in case somethings fails + // during container creation of startup. The podID (and containerID if container creation was + // successful) will be set to correct values so that the caller can correctly cleanup them even + // in case of a failure. + *podID = "" + *containerID = "" + *podID = runPodSandbox(t, client, ctx, sandboxRequest) + containerRequest.PodSandboxId = *podID + containerRequest.SandboxConfig = sandboxRequest.Config + *containerID = createContainer(t, client, ctx, containerRequest) + startContainer(t, client, ctx, *containerID) +} + +// Creates a template sandbox and then a template container inside it. +// Since, template container can take time to finish the init process and then exit (at which +// point it will actually be saved as a template) this function wait until the template is +// actually saved. +// It is the callers responsibility to clean the stop and remove the cloned +// containers and pods. +func createTemplateContainer(ctx context.Context, t *testing.T, client runtime.RuntimeServiceClient, templateSandboxRequest *runtime.RunPodSandboxRequest, templateContainerRequest *runtime.CreateContainerRequest, templatePodID, templateContainerID *string) { + createPodAndContainer(ctx, t, client, templateSandboxRequest, templateContainerRequest, templatePodID, templateContainerID) + + // Send a context with deadline for waitForTemplateSave function + d := time.Now().Add(10 * time.Second) + ctx, cancel := context.WithDeadline(ctx, d) + defer cancel() + waitForTemplateSave(ctx, t, *templatePodID) + return +} + +// Creates a clone from the given template pod and container. +// It is the callers responsibility to clean the stop and remove the cloned +// containers and pods. +func createClonedContainer(ctx context.Context, t *testing.T, client runtime.RuntimeServiceClient, templatePodID, templateContainerID string, cloneNumber int, clonedPodID, clonedContainerID *string) { + cloneSandboxRequest := getClonedPodConfig(cloneNumber, templatePodID) + cloneContainerRequest := getClonedContainerConfig(cloneNumber, templateContainerID) + createPodAndContainer(ctx, t, client, cloneSandboxRequest, cloneContainerRequest, clonedPodID, clonedContainerID) + return +} + +func verifyTemplateContainerState(ctx context.Context, t *testing.T, client runtime.RuntimeServiceClient, templateContainerID string) { + // sometimes it takes a little longer for containerd to update status of the template container + // so check it in a loop. + tries := 3 + var templateContainerState runtime.ContainerState + for i := 0; i < tries; i++ { + templateContainerState := getContainerStatus(t, client, ctx, templateContainerID) + if templateContainerState == runtime.ContainerState_CONTAINER_EXITED { + return + } + time.Sleep(500 * time.Millisecond) + } + t.Fatalf("template container %s expected state EXITED actual: %s", templateContainerID, runtime.ContainerState_name[int32(templateContainerState)]) +} + +// Runs a command inside given container and verifies if the command executes successfully. +func verifyContainerExec(ctx context.Context, t *testing.T, client runtime.RuntimeServiceClient, containerID string) { + execCommand := []string{ + "ping", + "www.microsoft.com", + } + + execRequest := &runtime.ExecSyncRequest{ + ContainerId: containerID, + Cmd: execCommand, + Timeout: 20, + } + + r := execSync(t, client, ctx, execRequest) + output := strings.TrimSpace(string(r.Stdout)) + errorMsg := string(r.Stderr) + exitCode := int(r.ExitCode) + + if exitCode != 0 || len(errorMsg) != 0 { + t.Fatalf("Failed execution inside container %s with error: %s, exitCode: %d", containerID, errorMsg, exitCode) + } else { + t.Logf("Exec(container: %s) stdout: %s, stderr: %s, exitCode: %d\n", containerID, output, errorMsg, exitCode) + } +} + +func cleanupPod(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, podID *string) { + if *podID == "" { + // Do nothing for empty podID + return + } + stopPodSandbox(t, client, ctx, *podID) + removePodSandbox(t, client, ctx, *podID) +} + +func cleanupContainer(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID *string) { + if *containerID == "" { + // Do nothing for empty containerID + return + } + stopContainer(t, client, ctx, *containerID) + removeContainer(t, client, ctx, *containerID) +} + +// A simple test to just create a template container and then create one +// cloned container from that template. +func Test_CloneContainer_WCOW(t *testing.T) { + requireFeatures(t, featureWCOWHypervisor) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := newTestRuntimeClient(t) + var templatePodID, templateContainerID string + var clonedPodID, clonedContainerID string + + // send pointers so that immediate evaluation of arguments doesn't evaluate to empty strings + defer cleanupPod(t, client, ctx, &templatePodID) + defer cleanupContainer(t, client, ctx, &templateContainerID) + defer cleanupPod(t, client, ctx, &clonedPodID) + defer cleanupContainer(t, client, ctx, &clonedContainerID) + + pullRequiredImages(t, []string{imageWindowsNanoserver}) + + createTemplateContainer(ctx, t, client, getTemplatePodConfig("templatepod"), getTemplateContainerConfig("templatecontainer"), &templatePodID, &templateContainerID) + verifyTemplateContainerState(ctx, t, client, templateContainerID) + createClonedContainer(ctx, t, client, templatePodID, templateContainerID, 1, &clonedPodID, &clonedContainerID) + verifyContainerExec(ctx, t, client, clonedContainerID) +} + +// A test for creating multiple clones(3 clones) from one template container. +func Test_MultiplClonedContainers_WCOW(t *testing.T) { + requireFeatures(t, featureWCOWHypervisor) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := newTestRuntimeClient(t) + nClones := 3 + var templatePodID, templateContainerID string + clonedPodIDs := make([]string, nClones) + clonedContainerIDs := make([]string, nClones) + + defer cleanupPod(t, client, ctx, &templatePodID) + defer cleanupContainer(t, client, ctx, &templateContainerID) + for i := 0; i < nClones; i++ { + defer cleanupPod(t, client, ctx, &clonedPodIDs[i]) + defer cleanupContainer(t, client, ctx, &clonedContainerIDs[i]) + } + + pullRequiredImages(t, []string{imageWindowsNanoserver}) + + // create template pod & container + createTemplateContainer(ctx, t, client, getTemplatePodConfig("templatepod"), getTemplateContainerConfig("templatecontainer"), &templatePodID, &templateContainerID) + verifyTemplateContainerState(ctx, t, client, templateContainerID) + + // create multiple clones + for i := 0; i < nClones; i++ { + createClonedContainer(ctx, t, client, templatePodID, templateContainerID, i, &clonedPodIDs[i], &clonedContainerIDs[i]) + } + + for i := 0; i < nClones; i++ { + verifyContainerExec(ctx, t, client, clonedContainerIDs[i]) + } +} + +// Test if a normal container can be created inside a clond pod alongside the cloned +// container. +func Test_NormalContainerInClonedPod_WCOW(t *testing.T) { + requireFeatures(t, featureWCOWHypervisor) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := newTestRuntimeClient(t) + var templatePodID, templateContainerID string + var clonedPodID, clonedContainerID string + var stdContainerID string + pullRequiredImages(t, []string{imageWindowsNanoserver}) + + defer cleanupPod(t, client, ctx, &templatePodID) + defer cleanupContainer(t, client, ctx, &templateContainerID) + defer cleanupPod(t, client, ctx, &clonedPodID) + defer cleanupContainer(t, client, ctx, &clonedContainerID) + defer cleanupContainer(t, client, ctx, &stdContainerID) + + // create template pod & container + createTemplateContainer(ctx, t, client, getTemplatePodConfig("templatepod"), getTemplateContainerConfig("templatecontainer"), &templatePodID, &templateContainerID) + verifyTemplateContainerState(ctx, t, client, templateContainerID) + + // create a cloned pod and a cloned container + cloneSandboxRequest := getClonedPodConfig(1, templatePodID) + cloneContainerRequest := getClonedContainerConfig(1, templateContainerID) + createPodAndContainer(ctx, t, client, cloneSandboxRequest, cloneContainerRequest, &clonedPodID, &clonedContainerID) + + // create a normal container in cloned pod + stdContainerRequest := getStandardContainerConfig("standard-container") + stdContainerRequest.PodSandboxId = clonedPodID + stdContainerRequest.SandboxConfig = cloneSandboxRequest.Config + stdContainerID = createContainer(t, client, ctx, stdContainerRequest) + startContainer(t, client, ctx, stdContainerID) + + verifyContainerExec(ctx, t, client, clonedContainerID) + verifyContainerExec(ctx, t, client, stdContainerID) +} + +// A test for cloning multiple pods first and then cloning one container in each +// of those pods. +func Test_CloneContainersWithClonedPodPool_WCOW(t *testing.T) { + requireFeatures(t, featureWCOWHypervisor) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := newTestRuntimeClient(t) + nClones := 3 + var templatePodID, templateContainerID string + clonedPodIDs := make([]string, nClones) + clonedContainerIDs := make([]string, nClones) + + defer cleanupPod(t, client, ctx, &templatePodID) + defer cleanupContainer(t, client, ctx, &templateContainerID) + for i := 0; i < nClones; i++ { + defer cleanupPod(t, client, ctx, &clonedPodIDs[i]) + defer cleanupContainer(t, client, ctx, &clonedContainerIDs[i]) + } + + pullRequiredImages(t, []string{imageWindowsNanoserver}) + + // create template pod & container + createTemplateContainer(ctx, t, client, getTemplatePodConfig("templatepod"), getTemplateContainerConfig("templatecontainer"), &templatePodID, &templateContainerID) + verifyTemplateContainerState(ctx, t, client, templateContainerID) + + // create multiple pods + clonedSandboxRequests := []*runtime.RunPodSandboxRequest{} + for i := 0; i < nClones; i++ { + cloneSandboxRequest := getClonedPodConfig(i, templatePodID) + clonedPodIDs[i] = runPodSandbox(t, client, ctx, cloneSandboxRequest) + clonedSandboxRequests = append(clonedSandboxRequests, cloneSandboxRequest) + } + + // create multiple clones + for i := 0; i < nClones; i++ { + cloneContainerRequest := getClonedContainerConfig(i, templateContainerID) + + cloneContainerRequest.PodSandboxId = clonedPodIDs[i] + cloneContainerRequest.SandboxConfig = clonedSandboxRequests[i].Config + clonedContainerIDs[i] = createContainer(t, client, ctx, cloneContainerRequest) + startContainer(t, client, ctx, clonedContainerIDs[i]) + } + + for i := 0; i < nClones; i++ { + verifyContainerExec(ctx, t, client, clonedContainerIDs[i]) + } +} + +func Test_ClonedContainerRunningAfterDeletingTemplate(t *testing.T) { + requireFeatures(t, featureWCOWHypervisor) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := newTestRuntimeClient(t) + var templatePodID, templateContainerID string + var clonedPodID, clonedContainerID string + + // send pointers so that immediate evaluation of arguments doesn't evaluate to empty strings + defer cleanupPod(t, client, ctx, &templatePodID) + defer cleanupContainer(t, client, ctx, &templateContainerID) + defer cleanupPod(t, client, ctx, &clonedPodID) + defer cleanupContainer(t, client, ctx, &clonedContainerID) + + pullRequiredImages(t, []string{imageWindowsNanoserver}) + + createTemplateContainer(ctx, t, client, getTemplatePodConfig("templatepod"), getTemplateContainerConfig("templatecontainer"), &templatePodID, &templateContainerID) + verifyTemplateContainerState(ctx, t, client, templateContainerID) + + createClonedContainer(ctx, t, client, templatePodID, templateContainerID, 1, &clonedPodID, &clonedContainerID) + + stopPodSandbox(t, client, ctx, templatePodID) + removePodSandbox(t, client, ctx, templatePodID) + // Make sure cleanup function doesn't try to cleanup this deleted pod. + templatePodID = "" + templateContainerID = "" + + verifyContainerExec(ctx, t, client, clonedContainerID) +} + +// A test to verify that multiple templats can be created and clones +// can be made from each of them simultaneously. +func Test_MultipleTemplateAndClones_WCOW(t *testing.T) { + requireFeatures(t, featureWCOWHypervisor) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := newTestRuntimeClient(t) + nTemplates := 2 + var wg sync.WaitGroup + templatePodIDs := make([]string, nTemplates) + templateContainerIDs := make([]string, nTemplates) + clonedPodIDs := make([]string, nTemplates) + clonedContainerIDs := make([]string, nTemplates) + + for i := 0; i < nTemplates; i++ { + defer cleanupPod(t, client, ctx, &templatePodIDs[i]) + defer cleanupContainer(t, client, ctx, &templateContainerIDs[i]) + defer cleanupPod(t, client, ctx, &clonedPodIDs[i]) + defer cleanupContainer(t, client, ctx, &clonedContainerIDs[i]) + } + + pullRequiredImages(t, []string{imageWindowsNanoserver}) + + wg.Add(nTemplates) + for i := 0; i < nTemplates; i++ { + go func(index int) { + defer wg.Done() + createTemplateContainer(ctx, t, client, getTemplatePodConfig(fmt.Sprintf("templatepod-%d", index)), getTemplateContainerConfig(fmt.Sprintf("templatecontainer-%d", index)), &templatePodIDs[index], &templateContainerIDs[index]) + verifyTemplateContainerState(ctx, t, client, templateContainerIDs[index]) + createClonedContainer(ctx, t, client, templatePodIDs[index], templateContainerIDs[index], index, &clonedPodIDs[index], &clonedContainerIDs[index]) + }(i) + } + + // Wait before all template & clone creations are done. + wg.Wait() + + for i := 0; i < nTemplates; i++ { + verifyContainerExec(ctx, t, client, clonedContainerIDs[i]) + } +} + +// Tries to create a clone with a different memory config than the template +// and verifies that the request correctly fails with an error. +func Test_VerifyCloneAndTemplateConfig(t *testing.T) { + requireFeatures(t, featureWCOWHypervisor) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + client := newTestRuntimeClient(t) + var templatePodID, templateContainerID string + var clonedPodID string + + // send pointers so that immediate evaluation of arguments doesn't evaluate to empty strings + defer cleanupPod(t, client, ctx, &templatePodID) + defer cleanupContainer(t, client, ctx, &templateContainerID) + defer cleanupPod(t, client, ctx, &clonedPodID) + + pullRequiredImages(t, []string{imageWindowsNanoserver}) + + templatePodConfig := getTemplatePodConfig("templatepod") + + createTemplateContainer(ctx, t, client, templatePodConfig, getTemplateContainerConfig("templatecontainer"), &templatePodID, &templateContainerID) + verifyTemplateContainerState(ctx, t, client, templateContainerID) + + // change pod config to make sure the request fails + cloneSandboxRequest := getClonedPodConfig(0, templatePodID) + cloneSandboxRequest.Config.Annotations["io.microsoft.virtualmachine.computetopology.memory.allowovercommit"] = "false" + + _, err := client.RunPodSandbox(ctx, cloneSandboxRequest) + if err == nil { + t.Fatalf("pod cloning should fail with mismatching configurations error") + } else if !strings.Contains(err.Error(), "doesn't match") { + t.Fatalf("Expected mismatching configurations error got: %s", err) + } +} diff --git a/test/cri-containerd/container.go b/test/cri-containerd/container.go index af4137e691..1fbe5bb976 100644 --- a/test/cri-containerd/container.go +++ b/test/cri-containerd/container.go @@ -64,3 +64,13 @@ func getCreateContainerRequest(podID string, name string, image string, command SandboxConfig: podConfig, } } + +func getContainerStatus(t *testing.T, client runtime.RuntimeServiceClient, ctx context.Context, containerID string) runtime.ContainerState { + response, err := client.ContainerStatus(ctx, &runtime.ContainerStatusRequest{ + ContainerId: containerID, + }) + if err != nil { + t.Fatalf("failed ContainerStatus request for container: %s, with: %v", containerID, err) + } + return response.Status.State +}