From fcc18548da0562ab57feb895f5d1892e824853d1 Mon Sep 17 00:00:00 2001 From: Daniel Canter Date: Thu, 11 Mar 2021 11:04:17 -0800 Subject: [PATCH] Add new job container tests Add tests to excercise the scenarios we care about and expect to work. This includes: 1. HNS access. Validate we can create a dead simple network and remove after. 2. Look at disks/volumes/partitions on the host. 3. Create a vhd from in the container. 4. Check hostname at beginning of test -> exec in job container and check hostname -> compare to validate that they're the same. 4. Manipulating/generating etw file. 5. Install (and uninstall on success) a program. Signed-off-by: Daniel Canter --- test/cri-containerd/jobcontainer_test.go | 249 +++++++++++++++++- test/cri-containerd/main.go | 15 +- .../imagepull_timestamp}/Dockerfile | 0 .../jobcontainer_createvhd/Dockerfile | 11 + .../jobcontainer_createvhd/main.go | 36 +++ .../test-images/jobcontainer_etw/Dockerfile | 11 + .../test-images/jobcontainer_etw/main.go | 52 ++++ .../test-images/jobcontainer_etw/test.wprp | 38 +++ .../test-images/jobcontainer_hns/Dockerfile | 15 ++ .../test-images/jobcontainer_hns/main.go | 25 ++ 10 files changed, 442 insertions(+), 10 deletions(-) rename test/{testdata/pullimage => cri-containerd/test-images/imagepull_timestamp}/Dockerfile (100%) create mode 100644 test/cri-containerd/test-images/jobcontainer_createvhd/Dockerfile create mode 100644 test/cri-containerd/test-images/jobcontainer_createvhd/main.go create mode 100644 test/cri-containerd/test-images/jobcontainer_etw/Dockerfile create mode 100644 test/cri-containerd/test-images/jobcontainer_etw/main.go create mode 100644 test/cri-containerd/test-images/jobcontainer_etw/test.wprp create mode 100644 test/cri-containerd/test-images/jobcontainer_hns/Dockerfile create mode 100644 test/cri-containerd/test-images/jobcontainer_hns/main.go diff --git a/test/cri-containerd/jobcontainer_test.go b/test/cri-containerd/jobcontainer_test.go index 59e6dcc3c0..d1d0678dd8 100644 --- a/test/cri-containerd/jobcontainer_test.go +++ b/test/cri-containerd/jobcontainer_test.go @@ -4,10 +4,17 @@ package cri_containerd import ( "context" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" "strings" "testing" "time" + "github.com/Microsoft/go-winio/vhd" + "github.com/Microsoft/hcsshim/hcn" runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" ) @@ -26,14 +33,14 @@ func getJobContainerPodRequestWCOW(t *testing.T) *runtime.RunPodSandboxRequest { } } -func getJobContainerRequestWCOW(t *testing.T, podID string, podConfig *runtime.PodSandboxConfig) *runtime.CreateContainerRequest { +func getJobContainerRequestWCOW(t *testing.T, podID string, podConfig *runtime.PodSandboxConfig, image string) *runtime.CreateContainerRequest { return &runtime.CreateContainerRequest{ Config: &runtime.ContainerConfig{ Metadata: &runtime.ContainerMetadata{ Name: t.Name() + "-Container", }, Image: &runtime.ImageSpec{ - Image: imageWindowsNanoserver, + Image: image, }, Command: []string{ "cmd", @@ -44,7 +51,7 @@ func getJobContainerRequestWCOW(t *testing.T, podID string, podConfig *runtime.P }, Annotations: map[string]string{ - "microsoft.com/hostprocess": "true", + "microsoft.com/hostprocess-container": "true", "microsoft.com/hostprocess-inherit-user": "true", }, }, @@ -67,7 +74,7 @@ func Test_RunContainer_InheritUser_JobContainer_WCOW(t *testing.T) { defer removePodSandbox(t, client, podctx, podID) defer stopPodSandbox(t, client, podctx, podID) - containerRequest := getJobContainerRequestWCOW(t, podID, sandboxRequest.Config) + containerRequest := getJobContainerRequestWCOW(t, podID, sandboxRequest.Config, imageWindowsNanoserver) ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -85,3 +92,237 @@ func Test_RunContainer_InheritUser_JobContainer_WCOW(t *testing.T) { t.Fatalf("expected user: '%s', got '%s'", username, stdout) } } + +func Test_RunContainer_Hostname_JobContainer_WCOW(t *testing.T) { + requireFeatures(t, featureWCOWProcess, featureHostProcess) + + pullRequiredImages(t, []string{imageWindowsNanoserver}) + client := newTestRuntimeClient(t) + + // This test validates that the hostname we see on the host and in the container are the same, and they + // should be as the container is just a process on the host. + hostname, err := exec.Command("hostname").Output() + if err != nil { + t.Fatalf("failed to get hostname: %s", err) + } + + podctx := context.Background() + sandboxRequest := getJobContainerPodRequestWCOW(t) + + podID := runPodSandbox(t, client, podctx, sandboxRequest) + defer removePodSandbox(t, client, podctx, podID) + defer stopPodSandbox(t, client, podctx, podID) + + containerRequest := getJobContainerRequestWCOW(t, podID, sandboxRequest.Config, imageWindowsNanoserver) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + containerID := createContainer(t, client, ctx, containerRequest) + defer removeContainer(t, client, ctx, containerID) + startContainer(t, client, ctx, containerID) + defer stopContainer(t, client, ctx, containerID) + + execResponse := execSync(t, client, ctx, &runtime.ExecSyncRequest{ + ContainerId: containerID, + Cmd: []string{"hostname"}, + }) + containerStdout := strings.Trim(string(execResponse.Stdout), " \r\n") + hostStdout := strings.Trim(string(hostname), " \r\n") + if hostStdout != containerStdout { + t.Fatalf("expected hostname to be the same within job container. got %s but expected %s", hostStdout, containerStdout) + } +} + +func Test_RunContainer_HNS_JobContainer_WCOW(t *testing.T) { + requireFeatures(t, featureWCOWProcess, featureHostProcess) + + pullRequiredImages(t, []string{imageJobContainerHNS}) + client := newTestRuntimeClient(t) + + podctx := context.Background() + sandboxRequest := getJobContainerPodRequestWCOW(t) + + podID := runPodSandbox(t, client, podctx, sandboxRequest) + defer removePodSandbox(t, client, podctx, podID) + defer stopPodSandbox(t, client, podctx, podID) + + containerRequest := getJobContainerRequestWCOW(t, podID, sandboxRequest.Config, imageJobContainerHNS) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + networkName := fmt.Sprintf("JobContainer-Network-%s", podID) + containerRequest.Config.Command = []string{ + "go/src/hns/hns.exe", + networkName, + } + + containerID := createContainer(t, client, ctx, containerRequest) + defer removeContainer(t, client, ctx, containerID) + startContainer(t, client, ctx, containerID) + defer stopContainer(t, client, ctx, containerID) + + // Wait a couple seconds for the network to make sure the network is come up. + time.Sleep(time.Second * 5) + // After the init process ends, there should be an HNS network named after os.Args[1] that we passed + // in. Check if it exists to: + // 1. See if it worked and if it's not present we need to fail. + // 2. If it did work we need to delete it. + network, err := hcn.GetNetworkByName(networkName) + if err != nil { + if _, ok := err.(hcn.NetworkNotFoundError); ok { + t.Fatalf("no network/switch with name %q found: %s", networkName, err) + } + t.Fatalf("failed to get network/switch with name %q: %s", networkName, err) + } + + if err := network.Delete(); err != nil { + t.Fatalf("failed to delete HNS network: %s", err) + } +} + +func Test_RunContainer_VHD_JobContainer_WCOW(t *testing.T) { + requireFeatures(t, featureWCOWProcess, featureHostProcess) + + pullRequiredImages(t, []string{imageJobContainerVHD}) + client := newTestRuntimeClient(t) + + podctx := context.Background() + sandboxRequest := getJobContainerPodRequestWCOW(t) + + podID := runPodSandbox(t, client, podctx, sandboxRequest) + defer removePodSandbox(t, client, podctx, podID) + defer stopPodSandbox(t, client, podctx, podID) + + containerRequest := getJobContainerRequestWCOW(t, podID, sandboxRequest.Config, imageJobContainerVHD) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + + vhdPath := filepath.Join(dir, "test.vhdx") + containerRequest.Config.Command = []string{ + "go/src/vhd/vhd.exe", + vhdPath, + } + + containerID := createContainer(t, client, ctx, containerRequest) + defer removeContainer(t, client, ctx, containerID) + startContainer(t, client, ctx, containerID) + defer stopContainer(t, client, ctx, containerID) + + // Wait a couple seconds for the container to start up and make the vhd. + time.Sleep(time.Second * 3) + // The vhd.exe binary in the container will create an NTFS formatted vhd at `vhdPath`. Verify this + // exists and we can attach it. This is our success case. + if _, err := os.Stat(vhdPath); os.IsNotExist(err) { + t.Fatalf("vhd not present at %q: %s", vhdPath, err) + } + + if err := vhd.AttachVhd(vhdPath); err != nil { + t.Fatalf("failed to attach vhd at %q: %s", vhdPath, err) + } + + if err := vhd.DetachVhd(vhdPath); err != nil { + t.Fatalf("failed to detach vhd at %q: %s", vhdPath, err) + } +} + +func Test_RunContainer_ETW_JobContainer_WCOW(t *testing.T) { + requireFeatures(t, featureWCOWProcess, featureHostProcess) + + pullRequiredImages(t, []string{imageJobContainerETW}) + client := newTestRuntimeClient(t) + + podctx := context.Background() + sandboxRequest := getJobContainerPodRequestWCOW(t) + + podID := runPodSandbox(t, client, podctx, sandboxRequest) + defer removePodSandbox(t, client, podctx, podID) + defer stopPodSandbox(t, client, podctx, podID) + + containerRequest := getJobContainerRequestWCOW(t, podID, sandboxRequest.Config, imageJobContainerETW) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + // For this test we'll launch an image that has a wprp file inside that we'll use to take an etw trace. + // After the etl file is generated we'll use tracerpt to create a report/dump file of the trace. This is + // just to verify that a common use case of grabbing host traces/diagnostics can be achieved. + dir, err := ioutil.TempDir("", "") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + // Need for network name is solely because the only provider defined in the image is for HNS, so + // we do a simple HNS operation to get some output. + var ( + networkName = fmt.Sprintf("JobContainer-Network-%s", podID) + etlFile = filepath.Join(dir, "output.etl") + dumpFile = filepath.Join(dir, "output.xml") + ) + containerRequest.Config.Command = []string{ + "go/src/etw/etw.exe", + networkName, + etlFile, + dumpFile, + } + + containerID := createContainer(t, client, ctx, containerRequest) + defer removeContainer(t, client, ctx, containerID) + startContainer(t, client, ctx, containerID) + defer stopContainer(t, client, ctx, containerID) + + // Wait some time for the container to start up and manipulate the etl file. + time.Sleep(time.Second * 10) + if _, err := os.Stat(etlFile); os.IsNotExist(err) { + t.Fatalf("failed to find etl file %q: %s", etlFile, err) + } + + if _, err := os.Stat(dumpFile); os.IsNotExist(err) { + t.Fatalf("failed to find dump file %q: %s", dumpFile, err) + } +} + +func Test_RunContainer_HostVolumes_JobContainer_WCOW(t *testing.T) { + requireFeatures(t, featureWCOWProcess, featureHostProcess) + + pullRequiredImages(t, []string{imageWindowsNanoserver}) + client := newTestRuntimeClient(t) + + podctx := context.Background() + sandboxRequest := getJobContainerPodRequestWCOW(t) + + podID := runPodSandbox(t, client, podctx, sandboxRequest) + defer removePodSandbox(t, client, podctx, podID) + defer stopPodSandbox(t, client, podctx, podID) + + containerRequest := getJobContainerRequestWCOW(t, podID, sandboxRequest.Config, imageWindowsNanoserver) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + + containerID := createContainer(t, client, ctx, containerRequest) + defer removeContainer(t, client, ctx, containerID) + startContainer(t, client, ctx, containerID) + defer stopContainer(t, client, ctx, containerID) + + execResponse := execSync(t, client, ctx, &runtime.ExecSyncRequest{ + ContainerId: containerID, + Cmd: []string{"mountvol"}, + }) + containerStdout := strings.Trim(string(execResponse.Stdout), " \r\n") + + // This test validates we see the same volumes on the host as in the container. We have to do this after the + // container has been launched as the containers scratch space is a new volume + volumes, err := exec.Command("mountvol").Output() + if err != nil { + t.Fatalf("failed to get volumes: %s", err) + } + hostStdout := strings.Trim(string(volumes), " \r\n") + + if hostStdout != containerStdout { + t.Fatalf("expected volumes to be the same within job process container. got %q but expected %q", hostStdout, containerStdout) + } +} diff --git a/test/cri-containerd/main.go b/test/cri-containerd/main.go index c0f86f782a..64563d1788 100644 --- a/test/cri-containerd/main.go +++ b/test/cri-containerd/main.go @@ -39,12 +39,15 @@ const ( testDriversPath = "C:\\ContainerPlat\\testdrivers" testGPUBootFiles = "C:\\ContainerPlat\\LinuxBootFiles\\nvidiagpu" - lcowRuntimeHandler = "runhcs-lcow" - imageLcowK8sPause = "k8s.gcr.io/pause:3.1" - imageLcowAlpine = "docker.io/library/alpine:latest" - imageLcowCosmos = "cosmosarno/spark-master:2.4.1_2019-04-18_8e864ce" - alpineAspNet = "mcr.microsoft.com/dotnet/core/aspnet:3.1-alpine3.11" - alpineAspnetUpgrade = "mcr.microsoft.com/dotnet/core/aspnet:3.1.2-alpine3.11" + lcowRuntimeHandler = "runhcs-lcow" + imageLcowK8sPause = "k8s.gcr.io/pause:3.1" + imageLcowAlpine = "docker.io/library/alpine:latest" + imageLcowCosmos = "cosmosarno/spark-master:2.4.1_2019-04-18_8e864ce" + imageJobContainerHNS = "cplatpublic.azurecr.io/jobcontainer_hns:latest" + imageJobContainerETW = "cplatpublic.azurecr.io/jobcontainer_etw:latest" + imageJobContainerVHD = "cplatpublic.azurecr.io/jobcontainer_vhd:latest" + alpineAspNet = "mcr.microsoft.com/dotnet/core/aspnet:3.1-alpine3.11" + alpineAspnetUpgrade = "mcr.microsoft.com/dotnet/core/aspnet:3.1.2-alpine3.11" // Default account name for use with GMSA related tests. This will not be // present/you will not have access to the account on your machine unless // your environment is configured properly. diff --git a/test/testdata/pullimage/Dockerfile b/test/cri-containerd/test-images/imagepull_timestamp/Dockerfile similarity index 100% rename from test/testdata/pullimage/Dockerfile rename to test/cri-containerd/test-images/imagepull_timestamp/Dockerfile diff --git a/test/cri-containerd/test-images/jobcontainer_createvhd/Dockerfile b/test/cri-containerd/test-images/jobcontainer_createvhd/Dockerfile new file mode 100644 index 0000000000..189ab2dd8d --- /dev/null +++ b/test/cri-containerd/test-images/jobcontainer_createvhd/Dockerfile @@ -0,0 +1,11 @@ +# Irrelevant what image version we use for job containers as there's no container <-> host OS version restraint. +FROM golang:1.15.10-nanoserver-1809 + +# Get administrator privileges +USER containeradministrator + +WORKDIR /go/src/vhd +COPY main.go . + +RUN go get -d -v ./... +RUN go build -mod=mod \ No newline at end of file diff --git a/test/cri-containerd/test-images/jobcontainer_createvhd/main.go b/test/cri-containerd/test-images/jobcontainer_createvhd/main.go new file mode 100644 index 0000000000..1788dea39f --- /dev/null +++ b/test/cri-containerd/test-images/jobcontainer_createvhd/main.go @@ -0,0 +1,36 @@ +package main + +import ( + "context" + "log" + "os" + "syscall" + + "github.com/Microsoft/go-winio/vhd" + "github.com/Microsoft/hcsshim/computestorage" + "golang.org/x/sys/windows" +) + +// Simple binary to create a vhd with a single NTFS partition. +func main() { + if len(os.Args) < 2 { + log.Fatal("must provide VHDX name") + } + + vhdPath := os.Args[1] + if err := vhd.CreateVhdx(vhdPath, 1, 1); err != nil { + log.Fatalf("failed to create VHDX: %s", err) + } + + vhdHandle, err := vhd.OpenVirtualDisk(vhdPath, vhd.VirtualDiskAccessNone, vhd.OpenVirtualDiskFlagNone) + if err != nil { + log.Fatalf("failed to open VHDX: %s", err) + } + defer syscall.CloseHandle(vhdHandle) + + if err := computestorage.FormatWritableLayerVhd(context.Background(), windows.Handle(vhdHandle)); err != nil { + log.Fatalf("failed to format VHXD: %s", err) + } + + os.Exit(0) +} diff --git a/test/cri-containerd/test-images/jobcontainer_etw/Dockerfile b/test/cri-containerd/test-images/jobcontainer_etw/Dockerfile new file mode 100644 index 0000000000..8d860ac273 --- /dev/null +++ b/test/cri-containerd/test-images/jobcontainer_etw/Dockerfile @@ -0,0 +1,11 @@ +# Irrelevant what image version we use for job containers as there's no container <-> host OS version restraint. +FROM golang:1.15.10-nanoserver-1809 + +# Get administrator privileges +USER containeradministrator + +WORKDIR /go/src/etw +COPY . . + +RUN go get -d -v ./... +RUN go build -mod=mod \ No newline at end of file diff --git a/test/cri-containerd/test-images/jobcontainer_etw/main.go b/test/cri-containerd/test-images/jobcontainer_etw/main.go new file mode 100644 index 0000000000..4637d6160f --- /dev/null +++ b/test/cri-containerd/test-images/jobcontainer_etw/main.go @@ -0,0 +1,52 @@ +package main + +import ( + "log" + "os" + "os/exec" + "path/filepath" + + "github.com/Microsoft/hcsshim/hcn" +) + +func main() { + if len(os.Args) < 4 { + os.Exit(1) + } + + path := filepath.Join(filepath.Dir(os.Args[0]), "Test.wprp") + if err := exec.Command("wpr", "-start", path).Run(); err != nil { + log.Fatalf("failed to wpr start: %s", err) + } + + var ( + netName = os.Args[1] + etlFile = os.Args[2] + dumpFile = os.Args[3] + ) + + network := &hcn.HostComputeNetwork{ + Name: netName, + Type: hcn.NAT, + SchemaVersion: hcn.SchemaVersion{ + Major: 2, + Minor: 2, + }, + } + network, err := network.Create() + if err != nil { + log.Fatalf("failed to create hns network: %s", err) + } + + if err := network.Delete(); err != nil { + log.Fatalf("failed to delete hns network: %s", err) + } + + if err := exec.Command("wpr", "-stop", etlFile).Run(); err != nil { + log.Fatal(err) + } + + if err := exec.Command("tracerpt", etlFile, "-o", dumpFile, "-of", "XML").Run(); err != nil { + log.Fatal(err) + } +} diff --git a/test/cri-containerd/test-images/jobcontainer_etw/test.wprp b/test/cri-containerd/test-images/jobcontainer_etw/test.wprp new file mode 100644 index 0000000000..fdcd547461 --- /dev/null +++ b/test/cri-containerd/test-images/jobcontainer_etw/test.wprp @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/test/cri-containerd/test-images/jobcontainer_hns/Dockerfile b/test/cri-containerd/test-images/jobcontainer_hns/Dockerfile new file mode 100644 index 0000000000..3c0f8b990d --- /dev/null +++ b/test/cri-containerd/test-images/jobcontainer_hns/Dockerfile @@ -0,0 +1,15 @@ +# This dockerfile builds a super barebones container image that includes a binary to do a single HNS operation to +# validate that we can actually talk to HNS in a job container. As this is a huge usecase for job containers this is paramount +# to test. The binary in the image will NOT function if this image is used for a normal windows container, both process and hypervisor isolated. + +# Irrelevant what image version we use for job containers as there's no container <-> host OS version restraint. +FROM golang:1.15.10-nanoserver-1809 + +# Get administrator privileges +USER containeradministrator + +WORKDIR /go/src/hns +COPY main.go . + +RUN go get -d -v ./... +RUN go build -mod=mod \ No newline at end of file diff --git a/test/cri-containerd/test-images/jobcontainer_hns/main.go b/test/cri-containerd/test-images/jobcontainer_hns/main.go new file mode 100644 index 0000000000..e2e28f8d6b --- /dev/null +++ b/test/cri-containerd/test-images/jobcontainer_hns/main.go @@ -0,0 +1,25 @@ +package main + +import ( + "os" + + "github.com/Microsoft/hcsshim/hcn" +) + +func main() { + if len(os.Args) < 2 { + os.Exit(1) + } + network := &hcn.HostComputeNetwork{ + Name: os.Args[1], + Type: hcn.NAT, + SchemaVersion: hcn.SchemaVersion{ + Major: 2, + Minor: 2, + }, + } + network, err := network.Create() + if err != nil { + os.Exit(1) + } +}