diff --git a/scripts/Test-LCOW-UVM.ps1 b/scripts/Test-LCOW-UVM.ps1 new file mode 100644 index 0000000000..fbea64e2f5 --- /dev/null +++ b/scripts/Test-LCOW-UVM.ps1 @@ -0,0 +1,148 @@ +#ex: .\scripts\Test-LCOW-UVM.ps1 -vb -Action Bench -BootFilesPath C:\ContainerPlat\LinuxBootFiles\ -MountGCSTest -Count 2 -Benchtime '3s' +# benchstat via `go install golang.org/x/perf/cmd/benchstat@latest` + +[CmdletBinding()] +param ( + [ValidateSet('Test', 'Bench', 'List', 'Shell')] + [alias('a')] + [string] + $Action = 'Bench', + + [string] + $Note = '', + + # test parameters + [int] + $Count = 1, + + [string] + $BenchTime = '5s', + + [string] + $Timeout = '10m', + + [alias('tv')] + [switch] + $TestVerbose, + + [string] + $Run = '', + + [string] + $CodePath = '.', + + [string] + $OutDirectory = '.\test\results', + + # uvm parameters + + [string] + $BootFilesPath = 'C:\ContainerPlat\LinuxBootFiles', + + [ValidateSet('vhd', 'initrd')] + [string] + $BootFSType = 'vhd', + + [switch] + $DisableTimeSync, + + # gcs test/container options + + [string] + $ContainerRootFSMount = '/run/rootfs', + + [string] + $ContainerRootFSPath = (Join-Path $BootFilesPath 'rootfs.vhd'), + + [string] + $GCSTestMount = '/run/bin', + + [string] + $GCSTestPath = '.\bin\test\gcs.test', + + [switch] + $MountGCSTest, + + [string] + $Feature = '' +) + +Import-Module ( Join-Path $PSScriptRoot Testing.psm1 ) -Force + +$CodePath = Resolve-Path $CodePath +$OutDirectory = Resolve-Path $OutDirectory +$BootFilesPath = Resolve-Path $BootFilesPath +$ContainerRootFSPath = Resolve-Path $ContainerRootFSPath +$GCSTestPath = Resolve-Path $GCSTestPath + +$shell = ( $Action -eq 'Shell' ) + +if ( $shell ) { + $cmd = 'ash' +} else { + $date = Get-Date + $waitfiles = "$ContainerRootFSMount" + $gcspath = 'gcs.test' + if ( $MountGCSTest ) { + $waitfiles += ",$GCSTestMount" + $gcspath = "$GCSTestMount/gcs.test" + } + + $pre = "wait-paths -p $waitfiles -t 5 ; " + ` + 'echo nproc: `$(nproc) ; ' + ` + 'echo kernel: `$(uname -a) ; ' + ` + 'echo gcs.commit: `$(cat /info/gcs.commit 2>/dev/null) ; ' + ` + 'echo gcs.branch: `$(cat /info/gcs.branch 2>/dev/null) ; ' + ` + 'echo tar.date: `$(cat /info/tar.date 2>/dev/null) ; ' + ` + 'echo image.name: `$(cat /info/image.name 2>/dev/null) ; ' + ` + 'echo build.date: `$(cat /info/build.date 2>/dev/null) ; ' + + $testcmd, $out = New-TestCommand ` + -Action $Action ` + -Path $gcspath ` + -Name gcstest ` + -OutDirectory $OutDirectory ` + -Date $date ` + -Note $Note ` + -TestVerbose:$TestVerbose ` + -Count $Count ` + -BenchTime $BenchTime ` + -Timeout $Timeout ` + -Run $Run ` + -Feature $Feature ` + -Verbose:$Verbose + + $testcmd += " `'-rootfs-path=$ContainerRootFSMount`' " + $cmd = $pre + $testcmd +} + +$boot = '.\bin\tool\uvmboot.exe -gcs lcow ' + ` + '-fwd-stdout -fwd-stderr -output-handling stdout ' + ` + "-boot-files-path $BootFilesPath " + ` + "-root-fs-type $BootFSType " + ` + '-kernel-file vmlinux ' + ` + "-mount-scsi `"$ContainerRootFSPath,$ContainerRootFSMount`" " + +if ( $MountGCSTest ) { + $boot += "-share `"$GCSTestPath,$GCSTestMount`" " +} + +if ( $DisableTimeSync ) { + $boot += ' -disable-time-sync ' +} + +if ( $shell ) { + $boot += ' -t ' +} + +$boot += " -exec `"$cmd`" " + +Invoke-TestCommand ` + -TestCmd $boot ` + -TestCmdPreamble $testcmd ` + -OutputFile (&{ if ( $Action -ne 'Shell' ) { $out } }) ` + -OutputCmd (&{ if ( $Action -eq 'Bench' ) { 'benchstat' } }) ` + -Preamble ` + -Date $Date ` + -Note $Note ` + -Verbose:$Verbose diff --git a/test/gcs/container_bench_test.go b/test/gcs/container_bench_test.go new file mode 100644 index 0000000000..160b8f6ee6 --- /dev/null +++ b/test/gcs/container_bench_test.go @@ -0,0 +1,226 @@ +//go:build linux + +package gcs + +import ( + "context" + "testing" + + "github.com/Microsoft/hcsshim/internal/guest/prot" + "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" + "github.com/Microsoft/hcsshim/internal/guest/stdio" + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + cri_util "github.com/containerd/containerd/pkg/cri/util" + + testoci "github.com/Microsoft/hcsshim/test/internal/oci" +) + +func BenchmarkContainerCreate(b *testing.B) { + requireFeatures(b, featureStandalone) + ctx := namespaces.WithNamespace(context.Background(), testoci.DefaultNamespace) + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := b.Name() + cri_util.GenerateID() + scratch, rootfs := mountRootfs(ctx, b, host, id) + + s := testoci.CreateLinuxSpec(ctx, b, id, + testoci.DefaultLinuxSpecOpts(id, + oci.WithRootFSPath(rootfs), + oci.WithProcessArgs("/bin/sh", "-c", tailNull), + )..., + ) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: s, + } + + b.StartTimer() + c := createContainer(ctx, b, host, id, r) + b.StopTimer() + + // create launches background go-routines + // so kill container to end those and avoid future perf hits + killContainer(ctx, b, c) + deleteContainer(ctx, b, c) + removeContainer(ctx, b, host, id) + unmountRootfs(ctx, b, scratch) + } +} + +func BenchmarkContainerStart(b *testing.B) { + requireFeatures(b, featureStandalone) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := standaloneContainerRequest(ctx, b, host) + + c := createContainer(ctx, b, host, id, r) + + b.StartTimer() + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + b.StopTimer() + + killContainer(ctx, b, c) + waitContainer(ctx, b, c, p, true) + cleanupContainer(ctx, b, host, c) + cleanup() + } +} + +func BenchmarkContainerKill(b *testing.B) { + requireFeatures(b, featureStandalone) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := standaloneContainerRequest(ctx, b, host) + c := createContainer(ctx, b, host, id, r) + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + + b.StartTimer() + killContainer(ctx, b, c) + _, n := waitContainerRaw(c, p) + b.StopTimer() + + switch n { + case prot.NtForcedExit: + default: + b.Fatalf("container exit was %s", n) + } + + cleanupContainer(ctx, b, host, c) + cleanup() + } +} + +// benchmark container create through wait until exit. +func BenchmarkContainerCompleteExit(b *testing.B) { + requireFeatures(b, featureStandalone) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := standaloneContainerRequest(ctx, b, host, oci.WithProcessArgs("/bin/sh", "-c", "true")) + + b.StartTimer() + c := createContainer(ctx, b, host, id, r) + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + e, n := waitContainerRaw(c, p) + b.StopTimer() + + switch n { + case prot.NtGracefulExit, prot.NtUnexpectedExit: + default: + b.Fatalf("container exit was %s", n) + } + + if e != 0 { + b.Fatalf("container exit code was %d", e) + } + + killContainer(ctx, b, c) + c.Wait() + cleanupContainer(ctx, b, host, c) + cleanup() + } +} + +func BenchmarkContainerCompleteKill(b *testing.B) { + requireFeatures(b, featureStandalone) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := standaloneContainerRequest(ctx, b, host) + + b.StartTimer() + c := createContainer(ctx, b, host, id, r) + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + killContainer(ctx, b, c) + _, n := waitContainerRaw(c, p) + b.StopTimer() + + switch n { + case prot.NtForcedExit: + default: + b.Fatalf("container exit was %s", n) + } + + cleanupContainer(ctx, b, host, c) + cleanup() + } +} + +func BenchmarkContainerExec(b *testing.B) { + requireFeatures(b, featureStandalone) + ctx := namespaces.WithNamespace(context.Background(), testoci.DefaultNamespace) + host, _ := getTestState(ctx, b) + + id := b.Name() + c := createStandaloneContainer(ctx, b, host, id) + ip := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + ps := testoci.CreateLinuxSpec(ctx, b, id, + oci.WithDefaultPathEnv, + oci.WithProcessArgs("/bin/sh", "-c", "true"), + ).Process + + b.StartTimer() + p := execProcess(ctx, b, c, ps, stdio.ConnectionSettings{}) + exch, dch := p.Wait() + if e := <-exch; e != 0 { + b.Errorf("process exited with error code %d", e) + } + b.StopTimer() + + dch <- true + close(dch) + } + + killContainer(ctx, b, c) + waitContainer(ctx, b, c, ip, true) + cleanupContainer(ctx, b, host, c) +} + +func standaloneContainerRequest( + ctx context.Context, + t testing.TB, + host *hcsv2.Host, + extra ...oci.SpecOpts, +) (string, *prot.VMHostedContainerSettingsV2, func()) { + ctx = namespaces.WithNamespace(ctx, testoci.DefaultNamespace) + id := t.Name() + cri_util.GenerateID() + scratch, rootfs := mountRootfs(ctx, t, host, id) + + opts := testoci.DefaultLinuxSpecOpts(id, + oci.WithRootFSPath(rootfs), + oci.WithProcessArgs("/bin/sh", "-c", tailNull), + ) + opts = append(opts, extra...) + s := testoci.CreateLinuxSpec(ctx, t, id, opts...) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: s, + } + f := func() { + unmountRootfs(ctx, t, scratch) + } + + return id, r, f +} diff --git a/test/gcs/container_test.go b/test/gcs/container_test.go new file mode 100644 index 0000000000..6a68e55c33 --- /dev/null +++ b/test/gcs/container_test.go @@ -0,0 +1,237 @@ +//go:build linux + +package gcs + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + "golang.org/x/sync/errgroup" + + "github.com/Microsoft/hcsshim/internal/guest/gcserr" + "github.com/Microsoft/hcsshim/internal/guest/stdio" + + testoci "github.com/Microsoft/hcsshim/test/internal/oci" +) + +// +// tests for operations on standalone containers +// + +// todo: using `oci.WithTTY` for IO tests is broken and hangs + +func TestContainerCreate(t *testing.T) { + requireFeatures(t, featureStandalone) + + ctx := context.Background() + host, rtime := getTestState(ctx, t) + assertNumberContainers(ctx, t, rtime, 0) + + id := t.Name() + c := createStandaloneContainer(ctx, t, host, id) + t.Cleanup(func() { + cleanupContainer(ctx, t, host, c) + }) + + p := startContainer(ctx, t, c, stdio.ConnectionSettings{}) + t.Cleanup(func() { + killContainer(ctx, t, c) + waitContainer(ctx, t, c, p, true) + }) + + assertNumberContainers(ctx, t, rtime, 1) + css := listContainerStates(ctx, t, rtime) + // guaranteed by assertNumberContainers that css will only have 1 element + cs := css[0] + if cs.ID != id { + t.Fatalf("got id %q, wanted %q", cs.ID, id) + } + pid := p.Pid() + if pid != cs.Pid { + t.Fatalf("got pid %d, wanted %d", pid, cs.Pid) + } + if cs.Status != "running" { + t.Fatalf("got status %q, wanted %q", cs.Status, "running") + } +} + +func TestContainerDelete(t *testing.T) { + requireFeatures(t, featureStandalone) + + ctx := context.Background() + host, rtime := getTestState(ctx, t) + assertNumberContainers(ctx, t, rtime, 0) + + id := t.Name() + + c := createStandaloneContainer(ctx, t, host, id, + oci.WithProcessArgs("/bin/sh", "-c", "true"), + ) + + p := startContainer(ctx, t, c, stdio.ConnectionSettings{}) + waitContainer(ctx, t, c, p, false) + + cleanupContainer(ctx, t, host, c) + + _, err := host.GetCreatedContainer(id) + if hr, herr := gcserr.GetHresult(err); herr != nil || hr != gcserr.HrVmcomputeSystemNotFound { + t.Fatalf("GetCreatedContainer returned %v, wanted %v", err, gcserr.HrVmcomputeSystemNotFound) + } + assertNumberContainers(ctx, t, rtime, 0) +} + +// +// IO +// + +var ioTests = []struct { + name string + args []string + in string + want string +}{ + { + name: "true", + args: []string{"/bin/sh", "-c", "true"}, + want: "", + }, + { + name: "echo", + args: []string{"/bin/sh", "-c", `echo -n "hi y'all"`}, + want: "hi y'all", + }, + { + name: "tee", + args: []string{"/bin/sh", "-c", "tee"}, + in: "are you copying me?", + want: "are you copying me?", + }, +} + +func TestContainerIO(t *testing.T) { + requireFeatures(t, featureStandalone) + + ctx := context.Background() + host, rtime := getTestState(ctx, t) + assertNumberContainers(ctx, t, rtime, 0) + + for _, tt := range ioTests { + t.Run(tt.name, func(t *testing.T) { + id := strings.ReplaceAll(t.Name(), "/", "") + + con := newConnectionSettings(tt.in != "", true, true) + f := createStdIO(ctx, t, con) + + var outStr, errStr string + g := &errgroup.Group{} + g.Go(func() error { + outStr = f.ReadAllOut(ctx, t) + + return nil + }) + g.Go(func() error { + errStr = f.ReadAllErr(ctx, t) + + return nil + }) + + c := createStandaloneContainer(ctx, t, host, id, + oci.WithProcessArgs(tt.args...), + ) + t.Cleanup(func() { + cleanupContainer(ctx, t, host, c) + }) + p := startContainer(ctx, t, c, con) + + f.WriteIn(ctx, t, tt.in) + f.CloseIn(ctx, t) + t.Logf("wrote to stdin: %q", tt.in) + + waitContainer(ctx, t, c, p, false) + + _ = g.Wait() + t.Logf("stdout: %q", outStr) + t.Logf("stderr: %q", errStr) + + if errStr != "" { + t.Fatalf("container returned error %q", errStr) + } + if outStr != tt.want { + t.Fatalf("container returned %q; wanted %q", outStr, tt.want) + } + }) + } + + assertNumberContainers(ctx, t, rtime, 0) +} + +func TestContainerExec(t *testing.T) { + requireFeatures(t, featureStandalone) + + ctx := namespaces.WithNamespace(context.Background(), testoci.DefaultNamespace) + host, rtime := getTestState(ctx, t) + assertNumberContainers(ctx, t, rtime, 0) + + id := t.Name() + c := createStandaloneContainer(ctx, t, host, id) + t.Cleanup(func() { + cleanupContainer(ctx, t, host, c) + }) + + ip := startContainer(ctx, t, c, stdio.ConnectionSettings{}) + t.Cleanup(func() { + killContainer(ctx, t, c) + waitContainer(ctx, t, c, ip, true) + }) + + for _, tt := range ioTests { + t.Run(tt.name, func(t *testing.T) { + ps := testoci.CreateLinuxSpec(ctx, t, id, + oci.WithDefaultPathEnv, + oci.WithProcessArgs(tt.args...), + ).Process + con := newConnectionSettings(tt.in != "", true, true) + f := createStdIO(ctx, t, con) + + var outStr, errStr string + g := &errgroup.Group{} + g.Go(func() error { + outStr = f.ReadAllOut(ctx, t) + + return nil + }) + g.Go(func() error { + errStr = f.ReadAllErr(ctx, t) + + return nil + }) + + // OS pipes can lose some data, so sleep a bit to let ReadAll* kick off + time.Sleep(10 * time.Millisecond) + + p := execProcess(ctx, t, c, ps, con) + f.WriteIn(ctx, t, tt.in) + f.CloseIn(ctx, t) + t.Logf("wrote std in: %q", tt.in) + + exch, _ := p.Wait() + if i := <-exch; i != 0 { + t.Errorf("process exited with error code %d", i) + } + + _ = g.Wait() + t.Logf("stdout: %q", outStr) + t.Logf("stderr: %q", errStr) + + if errStr != "" { + t.Fatalf("exec returned error %q", errStr) + } else if outStr != tt.want { + t.Fatalf("process returned %q; wanted %q", outStr, tt.want) + } + }) + } +} diff --git a/test/gcs/cri_bench_test.go b/test/gcs/cri_bench_test.go new file mode 100644 index 0000000000..4f475e691b --- /dev/null +++ b/test/gcs/cri_bench_test.go @@ -0,0 +1,242 @@ +//go:build linux + +package gcs + +import ( + "context" + "testing" + + cri_util "github.com/containerd/containerd/pkg/cri/util" + + "github.com/Microsoft/hcsshim/internal/guest/prot" + "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" + "github.com/Microsoft/hcsshim/internal/guest/stdio" +) + +func BenchmarkCRISanboxCreate(b *testing.B) { + requireFeatures(b, featureCRI) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := cri_util.GenerateID() + scratch, rootfs := mountRootfs(ctx, b, host, id) + nns := id + createNamespace(ctx, b, nns) + spec := sandboxSpec(ctx, b, "test-bench-sandbox", id, nns, rootfs) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: spec, + } + + b.StartTimer() + c := createContainer(ctx, b, host, id, r) + b.StopTimer() + + // create launches background go-routines + // so kill container to end those and avoid future perf hits + killContainer(ctx, b, c) + cleanupContainer(ctx, b, host, c) + removeNamespace(ctx, b, nns) + unmountRootfs(ctx, b, scratch) + } +} + +func BenchmarkCRISandboxStart(b *testing.B) { + requireFeatures(b, featureCRI) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := cri_util.GenerateID() + scratch, rootfs := mountRootfs(ctx, b, host, id) + nns := id + createNamespace(ctx, b, nns) + spec := sandboxSpec(ctx, b, "test-bench-sandbox", id, nns, rootfs) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: spec, + } + + c := createContainer(ctx, b, host, id, r) + + b.StartTimer() + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + b.StopTimer() + + killContainer(ctx, b, c) + waitContainer(ctx, b, c, p, true) + cleanupContainer(ctx, b, host, c) + removeNamespace(ctx, b, nns) + unmountRootfs(ctx, b, scratch) + } +} + +func BenchmarkCRISandboxKill(b *testing.B) { + requireFeatures(b, featureCRI) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id := cri_util.GenerateID() + scratch, rootfs := mountRootfs(ctx, b, host, id) + nns := id + createNamespace(ctx, b, nns) + spec := sandboxSpec(ctx, b, "test-bench-sandbox", id, nns, rootfs) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: spec, + } + + c := createContainer(ctx, b, host, id, r) + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + + b.StartTimer() + killContainer(ctx, b, c) + _, n := waitContainerRaw(c, p) + b.StopTimer() + + switch n { + case prot.NtForcedExit: + default: + b.Fatalf("container exit was %s", n) + } + + cleanupContainer(ctx, b, host, c) + removeNamespace(ctx, b, nns) + unmountRootfs(ctx, b, scratch) + } +} + +func BenchmarkCRIWorkload(b *testing.B) { + requireFeatures(b, featureCRI) + ctx := context.Background() + host, _ := getTestState(ctx, b) + + sid := b.Name() + sScratch, sRootfs := mountRootfs(ctx, b, host, sid) + b.Cleanup(func() { + unmountRootfs(ctx, b, sScratch) + }) + nns := sid + createNamespace(ctx, b, nns) + b.Cleanup(func() { + removeNamespace(ctx, b, nns) + }) + + sSpec := sandboxSpec(ctx, b, "test-bench-sandbox", sid, nns, sRootfs) + sandbox := createContainer(ctx, b, host, sid, &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: sScratch, + OCISpecification: sSpec, + }) + b.Cleanup(func() { + cleanupContainer(ctx, b, host, sandbox) + }) + + sandboxInit := startContainer(ctx, b, sandbox, stdio.ConnectionSettings{}) + b.Cleanup(func() { + killContainer(ctx, b, sandbox) + waitContainer(ctx, b, sandbox, sandboxInit, true) + }) + + b.Run("Create", func(b *testing.B) { + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := workloadContainerRequest(ctx, b, host, sid, uint32(sandboxInit.Pid()), nns) + + b.StartTimer() + c := createContainer(ctx, b, host, id, r) + b.StopTimer() + + // create launches background go-routines + // so kill container to end those and avoid future perf hits + killContainer(ctx, b, c) + // edge case where workload container transitions from "created" to "paused" + // then "stopped" + waitContainerRaw(c, c.InitProcess()) + cleanupContainer(ctx, b, host, c) + cleanup() + } + }) + + b.Run("Start", func(b *testing.B) { + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := workloadContainerRequest(ctx, b, host, sid, uint32(sandboxInit.Pid()), nns) + c := createContainer(ctx, b, host, id, r) + + b.StartTimer() + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + b.StopTimer() + + killContainer(ctx, b, c) + waitContainer(ctx, b, c, p, true) + cleanupContainer(ctx, b, host, c) + cleanup() + } + }) + + b.Run("Kill", func(b *testing.B) { + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + id, r, cleanup := workloadContainerRequest(ctx, b, host, sid, uint32(sandboxInit.Pid()), nns) + c := createContainer(ctx, b, host, id, r) + p := startContainer(ctx, b, c, stdio.ConnectionSettings{}) + + b.StartTimer() + killContainer(ctx, b, c) + _, n := waitContainerRaw(c, p) + b.StopTimer() + + switch n { + case prot.NtForcedExit: + default: + b.Fatalf("container exit was %q, expected %q", n, prot.NtForcedExit) + } + + cleanupContainer(ctx, b, host, c) + cleanup() + } + }) +} + +func workloadContainerRequest( + ctx context.Context, + t testing.TB, + host *hcsv2.Host, + sid string, + spid uint32, + nns string, +) (string, *prot.VMHostedContainerSettingsV2, func()) { + id := sid + cri_util.GenerateID() + scratch, rootfs := mountRootfs(ctx, t, host, id) + spec := containerSpec(ctx, t, + sid, + spid, + "test-bench-container", + id, + []string{"/bin/sh", "-c"}, + []string{tailNull}, + "/", + nns, + rootfs, + ) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: spec, + } + f := func() { + unmountRootfs(ctx, t, scratch) + } + + return id, r, f +} diff --git a/test/gcs/cri_test.go b/test/gcs/cri_test.go new file mode 100644 index 0000000000..33f3a374f7 --- /dev/null +++ b/test/gcs/cri_test.go @@ -0,0 +1,98 @@ +//go:build linux + +package gcs + +import ( + "context" + "testing" + + "github.com/Microsoft/hcsshim/internal/guest/prot" + "github.com/Microsoft/hcsshim/internal/guest/stdio" +) + +// +// tests for operations on sandbox and workload (CRI) containers +// + +// TestCRILifecycle tests the entire CRI container workflow: creating and starting a CRI sandbox +// pod container, creating, starting, and stopping a workload container within that pod, asserting +// that all operations were successful, and mounting (and unmounting) rootfs's as necessary. +func TestCRILifecycle(t *testing.T) { + requireFeatures(t, featureCRI) + + ctx := context.Background() + host, rtime := getTestState(ctx, t) + assertNumberContainers(ctx, t, rtime, 0) + + sid := t.Name() + scratch, rootfs := mountRootfs(ctx, t, host, sid) + t.Cleanup(func() { + unmountRootfs(ctx, t, scratch) + }) + createNamespace(ctx, t, sid) + t.Cleanup(func() { + removeNamespace(ctx, t, sid) + }) + + spec := sandboxSpec(ctx, t, "test-sandbox", sid, sid, rootfs) + sandbox := createContainer(ctx, t, host, sid, &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: spec, + }) + t.Cleanup(func() { + cleanupContainer(ctx, t, host, sandbox) + assertNumberContainers(ctx, t, rtime, 0) + }) + + assertNumberContainers(ctx, t, rtime, 1) + assertContainerState(ctx, t, rtime, sid, "created") + + sandboxInit := startContainer(ctx, t, sandbox, stdio.ConnectionSettings{}) + t.Cleanup(func() { + killContainer(ctx, t, sandbox) + waitContainer(ctx, t, sandbox, sandboxInit, true) + }) + + assertContainerState(ctx, t, rtime, sid, "running") + cs := getContainerState(ctx, t, rtime, sid) + pid := sandboxInit.Pid() + if pid != cs.Pid { + t.Fatalf("got sandbox pid %d, wanted %d", pid, cs.Pid) + } + + cid := "container" + sid + cscratch, crootfs := mountRootfs(ctx, t, host, cid) + t.Cleanup(func() { + unmountRootfs(ctx, t, cscratch) + }) + + cspec := containerSpec(ctx, t, sid, uint32(sandboxInit.Pid()), "test-container", cid, + []string{"/bin/sh", "-c"}, + []string{tailNull}, + "/", sid, crootfs, + ) + workload := createContainer(ctx, t, host, cid, &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: cscratch, + OCISpecification: cspec, + }) + t.Cleanup(func() { + cleanupContainer(ctx, t, host, workload) + assertNumberContainers(ctx, t, rtime, 1) + }) + + assertNumberContainers(ctx, t, rtime, 2) + assertContainerState(ctx, t, rtime, cid, "created") + + workloadInit := startContainer(ctx, t, workload, stdio.ConnectionSettings{}) + assertContainerState(ctx, t, rtime, cid, "running") + t.Cleanup(func() { + killContainer(ctx, t, workload) + waitContainer(ctx, t, workload, workloadInit, true) + }) + + cs = getContainerState(ctx, t, rtime, cid) + pid = workloadInit.Pid() + if pid != cs.Pid { + t.Fatalf("got sandbox pid %d, wanted %d", pid, cs.Pid) + } +} diff --git a/test/gcs/doc.go b/test/gcs/doc.go new file mode 100644 index 0000000000..19d05fcc86 --- /dev/null +++ b/test/gcs/doc.go @@ -0,0 +1,3 @@ +// This package builds a test binary that can be run directly on uVM guest, +// alongside ./cmd/gcs, for testing and benchmarking. +package gcs diff --git a/test/gcs/helper_conn_test.go b/test/gcs/helper_conn_test.go new file mode 100644 index 0000000000..a80ee5d7f3 --- /dev/null +++ b/test/gcs/helper_conn_test.go @@ -0,0 +1,319 @@ +//go:build linux + +package gcs + +import ( + "context" + "errors" + "io" + "os" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" + + "github.com/Microsoft/hcsshim/internal/guest/stdio" + "github.com/Microsoft/hcsshim/internal/guest/transport" +) + +const ( + dialRetries = 4 + dialWait = 50 * time.Millisecond +) + +// port numbers to assign to connections. +var ( + _pipes sync.Map + _portNumber uint32 = 1 +) + +type PipeTransport struct{} + +var _ transport.Transport = &PipeTransport{} + +func (*PipeTransport) Dial(port uint32) (c transport.Connection, err error) { + for i := 0; i < dialRetries; i++ { + c, err = getFakeSocket(port) + + if errors.Is(err, unix.ENOENT) { + // socket hasn't been created + time.Sleep(dialWait) + continue + } + break + } + if err != nil { + return nil, err + } + + logrus.Debugf("dialed port %d", port) + return c, nil +} + +type fakeIO struct { + stdin, stdout, stderr *fakeSocket +} + +func createStdIO(ctx context.Context, t testing.TB, con stdio.ConnectionSettings) *fakeIO { + f := &fakeIO{} + if con.StdIn != nil { + f.stdin = newFakeSocket(ctx, t, *con.StdIn, "stdin") + } + if con.StdOut != nil { + f.stdout = newFakeSocket(ctx, t, *con.StdOut, "stdout") + } + if con.StdErr != nil { + f.stderr = newFakeSocket(ctx, t, *con.StdErr, "stderr") + } + + return f +} + +func (f *fakeIO) WriteIn(_ context.Context, t testing.TB, s string) { + if f.stdin == nil { + return + } + + b := []byte(s) + n := len(b) + + nn, err := f.stdin.Write(b) + if err != nil { + t.Helper() + t.Errorf("write to std in: %v", err) + } + if n != nn { + t.Helper() + t.Errorf("only wrote %d bytes, expected %d", nn, n) + } +} + +func (f *fakeIO) CloseIn(_ context.Context, t testing.TB) { + if f.stdin == nil { + return + } + + if err := f.stdin.CloseWrite(); err != nil { + t.Helper() + t.Errorf("close write std in: %v", err) + } + + if err := f.stdin.Close(); err != nil { + t.Helper() + t.Errorf("close std in: %v", err) + } +} + +func (f *fakeIO) ReadAllOut(ctx context.Context, t testing.TB) string { + return f.stdout.readAll(ctx, t) +} + +func (f *fakeIO) ReadAllErr(ctx context.Context, t testing.TB) string { + return f.stderr.readAll(ctx, t) +} + +type fakeSocket struct { + id uint32 + n string + ch chan struct{} // closed when dialed (via getFakeSocket) + r, w *os.File +} + +var _ transport.Connection = &fakeSocket{} + +func newFakeSocket(_ context.Context, t testing.TB, id uint32, n string) *fakeSocket { + t.Helper() + + _, ok := _pipes.Load(id) + if ok { + t.Fatalf("socket %d already exits", id) + } + + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("could not create socket: %v", err) + } + + s := &fakeSocket{ + id: id, + n: n, + r: r, + w: w, + ch: make(chan struct{}), + } + _pipes.Store(id, s) + + return s +} + +func getFakeSocket(id uint32) (*fakeSocket, error) { + f, ok := _pipes.Load(id) + if !ok { + return nil, unix.ENOENT + } + + s := f.(*fakeSocket) + select { + case <-s.ch: + default: + close(s.ch) + } + + return s, nil +} + +func (s *fakeSocket) Read(b []byte) (int, error) { + <-s.ch + return s.r.Read(b) +} + +func (s *fakeSocket) Write(b []byte) (int, error) { + <-s.ch + return s.w.Write(b) +} + +func (s *fakeSocket) Close() (err error) { + if _, ok := _pipes.LoadAndDelete(s.id); ok { + return nil + } + + err = s.r.Close() + if err := s.w.Close(); err != nil { + return err + } + + return err +} + +func (s *fakeSocket) CloseRead() error { + return s.r.Close() +} + +func (s *fakeSocket) CloseWrite() error { + return s.w.Close() +} + +func (*fakeSocket) File() (*os.File, error) { + return nil, errors.New("fakeSocket does not support File()") +} + +func (s *fakeSocket) readAll(ctx context.Context, t testing.TB) string { + return string(s.readAllByte(ctx, t)) +} + +func (s *fakeSocket) readAllByte(ctx context.Context, t testing.TB) (b []byte) { + if s == nil { + return nil + } + + var err error + ch := make(chan struct{}) + go func() { + defer close(ch) + b, err = io.ReadAll(s) + }() + + select { + case <-ch: + if err != nil { + t.Helper() + t.Errorf("read all %s: %v", s.n, err) + } + case <-ctx.Done(): + t.Helper() + t.Errorf("read all %s context cancelled: %v", s.n, ctx.Err()) + } + + return b +} + +func newConnectionSettings(in, out, err bool) stdio.ConnectionSettings { + c := stdio.ConnectionSettings{} + + if in { + p := nextPortNumber() + c.StdIn = &p + } + if out { + p := nextPortNumber() + c.StdOut = &p + } + if err { + p := nextPortNumber() + c.StdErr = &p + } + + return c +} + +func nextPortNumber() uint32 { + return atomic.AddUint32(&_portNumber, 2) +} + +func TestFakeSocket(t *testing.T) { + ctx := context.Background() + tpt := getTransport() + + ch := make(chan struct{}) + chs := make(chan struct{}) + con := newConnectionSettings(true, true, false) + + // host + f := createStdIO(ctx, t, con) + + var err error + go func() { // guest + defer close(ch) + + var cin, cout transport.Connection + cin, err = tpt.Dial(*con.StdIn) + if err != nil { + t.Logf("dial error %v", err) + + return + } + defer cin.Close() + + cout, err = tpt.Dial(*con.StdOut) + if err != nil { + t.Logf("dial error %v", err) + + return + } + defer cout.Close() + + close(chs) + var b []byte + b, err = io.ReadAll(cin) + if err != nil { + t.Logf("read all error: %v", err) + + return + } + t.Logf("guest read %s", b) + + _, err = cout.Write(b) + _ = cout.CloseWrite() + }() + + <-chs // wait for guest to dial + f.WriteIn(ctx, t, "hello") + f.WriteIn(ctx, t, " world") + f.CloseIn(ctx, t) + t.Logf("host wrote") + + <-ch + t.Logf("go routine closed") + if err != nil { + t.Fatalf("go routine error: %v", err) + } + + s := f.ReadAllOut(ctx, t) + t.Logf("host read %q", s) + if s != "hello world" { + t.Fatalf("got %q, wanted %q", s, "hello world") + } +} diff --git a/test/gcs/helper_container_test.go b/test/gcs/helper_container_test.go new file mode 100644 index 0000000000..1e95bb5ffd --- /dev/null +++ b/test/gcs/helper_container_test.go @@ -0,0 +1,282 @@ +//go:build linux + +package gcs + +import ( + "context" + "os" + "path/filepath" + "syscall" + "testing" + + "github.com/containerd/containerd/namespaces" + ctrdoci "github.com/containerd/containerd/oci" + oci "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/Microsoft/hcsshim/internal/guest/prot" + "github.com/Microsoft/hcsshim/internal/guest/runtime" + "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" + "github.com/Microsoft/hcsshim/internal/guest/stdio" + "github.com/Microsoft/hcsshim/internal/guest/storage" + "github.com/Microsoft/hcsshim/internal/guest/storage/overlay" + "github.com/Microsoft/hcsshim/internal/guestpath" + + testoci "github.com/Microsoft/hcsshim/test/internal/oci" +) + +// todo: autogenerate/fuzz realistic specs + +// +// testing helper functions for generic container management +// + +const tailNull = "tail -f /dev/null" + +// Creates an overlay mount, and then a container using that mount that runs until stopped. +// The container is created on its own, and not associated with a sandbox pod, and is therefore not CRI compliant. +// [unmountRootfs] is added to the test cleanup. +func createStandaloneContainer(ctx context.Context, t testing.TB, host *hcsv2.Host, id string, extra ...ctrdoci.SpecOpts) *hcsv2.Container { + ctx = namespaces.WithNamespace(ctx, testoci.DefaultNamespace) + scratch, rootfs := mountRootfs(ctx, t, host, id) + // spec is passed in from containerd and then updated in internal\hcsoci\create.go:CreateContainer() + opts := testoci.DefaultLinuxSpecOpts(id, + ctrdoci.WithRootFSPath(rootfs), + ctrdoci.WithProcessArgs("/bin/sh", "-c", tailNull), + ) + opts = append(opts, extra...) + s := testoci.CreateLinuxSpec(ctx, t, id, opts...) + r := &prot.VMHostedContainerSettingsV2{ + OCIBundlePath: scratch, + OCISpecification: s, + } + + t.Cleanup(func() { + unmountRootfs(ctx, t, scratch) + }) + + return createContainer(ctx, t, host, id, r) +} + +func createContainer(ctx context.Context, t testing.TB, host *hcsv2.Host, id string, s *prot.VMHostedContainerSettingsV2) *hcsv2.Container { + c, err := host.CreateContainer(ctx, id, s) + if err != nil { + t.Helper() + t.Fatalf("could not create container %q: %v", id, err) + } + + return c +} + +func removeContainer(_ context.Context, _ testing.TB, host *hcsv2.Host, id string) { + host.RemoveContainer(id) +} + +func startContainer(ctx context.Context, t testing.TB, c *hcsv2.Container, conn stdio.ConnectionSettings) hcsv2.Process { + pid, err := c.Start(ctx, conn) + if err != nil { + t.Helper() + t.Fatalf("could not start container %q: %v", c.ID(), err) + } + + return getProcess(ctx, t, c, uint32(pid)) +} + +// waitContainer waits on the container's init process, p. +func waitContainer(ctx context.Context, t testing.TB, c *hcsv2.Container, p hcsv2.Process, forced bool) { + t.Helper() + + var e int + ch := make(chan prot.NotificationType) + + // have to read the init process exit code to close the container + exch, dch := p.Wait() + defer close(dch) + go func() { + e = <-exch + dch <- true + ch <- c.Wait() + close(ch) + }() + + select { + case n, ok := <-ch: + if !ok { + t.Fatalf("container %q did not return a notification", c.ID()) + } + + switch { + // UnexpectedExit is the default, ForcedExit if killed + case n == prot.NtGracefulExit: + case n == prot.NtUnexpectedExit: + case forced && n == prot.NtForcedExit: + default: + t.Fatalf("container %q exited with %s", c.ID(), n) + } + case <-ctx.Done(): + t.Fatalf("context canceled: %v", ctx.Err()) + } + + switch { + case e == 0: + case forced && e == 137: + default: + t.Fatalf("got exit code %d", e) + } +} + +func waitContainerRaw(c *hcsv2.Container, p hcsv2.Process) (int, prot.NotificationType) { + exch, dch := p.Wait() + defer close(dch) + r := <-exch + dch <- true + n := c.Wait() + + return r, n +} + +func execProcess(ctx context.Context, t testing.TB, c *hcsv2.Container, p *oci.Process, con stdio.ConnectionSettings) hcsv2.Process { + pid, err := c.ExecProcess(ctx, p, con) + if err != nil { + t.Helper() + t.Fatalf("could not exec process: %v", err) + } + + return getProcess(ctx, t, c, uint32(pid)) +} + +func getProcess(_ context.Context, t testing.TB, c *hcsv2.Container, pid uint32) hcsv2.Process { + p, err := c.GetProcess(pid) + if err != nil { + t.Helper() + t.Fatalf("could not get process %d: %v", pid, err) + } + + return p +} + +func killContainer(ctx context.Context, t testing.TB, c *hcsv2.Container) { + if err := c.Kill(ctx, syscall.SIGKILL); err != nil { + t.Helper() + t.Fatalf("could not kill container %q: %v", c.ID(), err) + } +} + +func deleteContainer(ctx context.Context, t testing.TB, c *hcsv2.Container) { + if err := c.Delete(ctx); err != nil { + t.Helper() + t.Fatalf("could not delete container %q: %v", c.ID(), err) + } +} + +func cleanupContainer(ctx context.Context, t testing.TB, host *hcsv2.Host, c *hcsv2.Container) { + deleteContainer(ctx, t, c) + removeContainer(ctx, t, host, c.ID()) +} + +// +// runtime +// + +func listContainerStates(_ context.Context, t testing.TB, rt runtime.Runtime) []runtime.ContainerState { + css, err := rt.ListContainerStates() + if err != nil { + t.Helper() + t.Fatalf("could not list containers: %v", err) + } + + return css +} + +// assertNumberContainers asserts that n containers are found, and then returns the container states. +func assertNumberContainers(ctx context.Context, t testing.TB, rt runtime.Runtime, n int) { + fmt := "found %d running containers, wanted %d" + css := listContainerStates(ctx, t, rt) + nn := len(css) + if nn != n { + t.Helper() + + if nn == 0 { + t.Fatalf(fmt, nn, n) + } + + cs := make([]string, nn) + for i, c := range css { + cs[i] = c.ID + } + + t.Fatalf(fmt+":\n%#+v", nn, n, cs) + } +} + +func getContainerState(ctx context.Context, t testing.TB, rt runtime.Runtime, id string) runtime.ContainerState { + css := listContainerStates(ctx, t, rt) + + for _, cs := range css { + if cs.ID == id { + return cs + } + } + + t.Helper() + t.Fatalf("could not find container %q", id) + return runtime.ContainerState{} // just to make the linter happy +} + +func assertContainerState(ctx context.Context, t testing.TB, rt runtime.Runtime, id, state string) { + cs := getContainerState(ctx, t, rt, id) + if cs.Status != state { + t.Helper() + t.Fatalf("got container %q status %q, wanted %q", id, cs.Status, state) + } +} + +// +// mount management +// + +func mountRootfs(ctx context.Context, t testing.TB, host *hcsv2.Host, id string) (scratch string, rootfs string) { + scratch = filepath.Join(guestpath.LCOWRootPrefixInUVM, id) + rootfs = filepath.Join(scratch, "rootfs") + if err := overlay.MountLayer(ctx, + []string{*flagRootfsPath}, + filepath.Join(scratch, "upper"), + filepath.Join(scratch, "work"), + rootfs, + false, // readonly + id, + host.SecurityPolicyEnforcer(), + ); err != nil { + t.Helper() + t.Fatalf("could not mount overlay layers from %q: %v", *flagRootfsPath, err) + } + + return scratch, rootfs +} + +func unmountRootfs(ctx context.Context, t testing.TB, path string) { + if err := storage.UnmountAllInPath(ctx, path, true); err != nil { + t.Fatalf("could not unmount container rootfs: %v", err) + } + if err := os.RemoveAll(path); err != nil { + t.Fatalf("could not remove container directory: %v", err) + } +} + +// +// network namespaces +// + +func createNamespace(ctx context.Context, t testing.TB, nns string) { + ns := hcsv2.GetOrAddNetworkNamespace(nns) + if err := ns.Sync(ctx); err != nil { + t.Helper() + t.Fatalf("could not sync new namespace %q: %v", nns, err) + } +} + +func removeNamespace(ctx context.Context, t testing.TB, nns string) { + if err := hcsv2.RemoveNetworkNamespace(ctx, nns); err != nil { + t.Helper() + t.Fatalf("could not remove namespace %q: %v", nns, err) + } +} diff --git a/test/gcs/helper_cri_test.go b/test/gcs/helper_cri_test.go new file mode 100644 index 0000000000..cf9dd09147 --- /dev/null +++ b/test/gcs/helper_cri_test.go @@ -0,0 +1,134 @@ +//go:build linux + +package gcs + +import ( + "context" + "testing" + + "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" + "github.com/containerd/containerd/pkg/cri/annotations" + criopts "github.com/containerd/containerd/pkg/cri/opts" + + testoci "github.com/Microsoft/hcsshim/test/internal/oci" +) + +// +// testing helper functions for generic container management +// + +func sandboxSpec( + ctx context.Context, + t testing.TB, + name string, + id string, + nns string, + root string, + extra ...oci.SpecOpts, +) *oci.Spec { + ctx = namespaces.WithNamespace(ctx, testoci.CRINamespace) + opts := sandboxSpecOpts(ctx, t, name, id, nns, root) + opts = append(opts, extra...) + + return testoci.CreateLinuxSpec(ctx, t, id, opts...) +} + +func sandboxSpecOpts(_ context.Context, t testing.TB, + name string, + id string, + nns string, + root string, +) []oci.SpecOpts { + img := testoci.LinuxSandboxImageConfig(*flagSandboxPause) + cfg := testoci.LinuxSandboxRuntimeConfig(name) + + opts := testoci.DefaultLinuxSpecOpts(nns, + oci.WithEnv(img.Env), + oci.WithHostname(cfg.GetHostname()), + oci.WithRootFSPath(root), + ) + + if usr := img.User; usr != "" { + oci.WithUser(usr) + } + + if img.WorkingDir != "" { + opts = append(opts, oci.WithProcessCwd(img.WorkingDir)) + } + + if len(img.Entrypoint) == 0 && len(img.Cmd) == 0 { + t.Helper() + t.Fatalf("invalid empty entrypoint and cmd in image config %+v", img) + } + opts = append(opts, oci.WithProcessArgs(append(img.Entrypoint, img.Cmd...)...)) + + opts = append(opts, + criopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeSandbox), + criopts.WithAnnotation(annotations.SandboxID, id), + criopts.WithAnnotation(annotations.SandboxNamespace, cfg.GetMetadata().GetNamespace()), + criopts.WithAnnotation(annotations.SandboxName, cfg.GetMetadata().GetName()), + criopts.WithAnnotation(annotations.SandboxLogDir, cfg.GetLogDirectory()), + ) + + return opts +} + +func containerSpec( + ctx context.Context, + t testing.TB, + sandboxID string, + sandboxPID uint32, + name string, + id string, + cmd []string, + args []string, + wd string, + nns string, + root string, + extra ...oci.SpecOpts, +) *oci.Spec { + ctx = namespaces.WithNamespace(ctx, testoci.CRINamespace) + opts := containerSpecOpts(ctx, t, sandboxID, sandboxPID, name, cmd, args, wd, nns, root) + opts = append(opts, extra...) + + return testoci.CreateLinuxSpec(ctx, t, id, opts...) +} + +func containerSpecOpts(_ context.Context, _ testing.TB, + sandboxID string, + sandboxPID uint32, + name string, + cmd []string, + args []string, + wd string, + nns string, + root string, +) []oci.SpecOpts { + cfg := testoci.LinuxWorkloadRuntimeConfig(name, cmd, args, wd) + img := testoci.LinuxWorkloadImageConfig() + + opts := testoci.DefaultLinuxSpecOpts(nns, + oci.WithRootFSPath(root), + oci.WithEnv(nil), + // this will be set based on the security context below + oci.WithNewPrivileges, + criopts.WithProcessArgs(cfg, img), + criopts.WithPodNamespaces(nil, sandboxPID, sandboxPID), + ) + + hostname := name + env := append([]string{testoci.HostnameEnv + "=" + hostname}, img.Env...) + for _, e := range cfg.GetEnvs() { + env = append(env, e.GetKey()+"="+e.GetValue()) + } + opts = append(opts, oci.WithEnv(env)) + + opts = append(opts, + criopts.WithAnnotation(annotations.ContainerType, annotations.ContainerTypeContainer), + criopts.WithAnnotation(annotations.SandboxID, sandboxID), + criopts.WithAnnotation(annotations.ContainerName, name), + ) + + return opts +} diff --git a/test/gcs/main_test.go b/test/gcs/main_test.go new file mode 100644 index 0000000000..5ab9f9be6e --- /dev/null +++ b/test/gcs/main_test.go @@ -0,0 +1,167 @@ +//go:build linux + +package gcs + +import ( + "context" + "flag" + "fmt" + "log" + "os" + "strconv" + "testing" + + "github.com/containerd/cgroups" + "github.com/sirupsen/logrus" + + "github.com/Microsoft/hcsshim/internal/guest/runtime" + "github.com/Microsoft/hcsshim/internal/guest/runtime/hcsv2" + "github.com/Microsoft/hcsshim/internal/guest/runtime/runc" + "github.com/Microsoft/hcsshim/internal/guest/transport" + "github.com/Microsoft/hcsshim/internal/guestpath" + "github.com/Microsoft/hcsshim/pkg/securitypolicy" + + testflag "github.com/Microsoft/hcsshim/test/internal/flag" + "github.com/Microsoft/hcsshim/test/internal/require" +) + +const ( + featureCRI = "CRI" + featureStandalone = "StandAlone" +) + +var allFeatures = []string{ + featureCRI, + featureStandalone, +} + +var ( + flagFeatures = testflag.NewFeatureFlag(allFeatures) + flagJoinGCSCgroup = flag.Bool( + "join-gcs-cgroup", + false, + "If true, join the same cgroup as the gcs daemon, `/gcs`", + ) + flagRootfsPath = flag.String( + "rootfs-path", + "/run/rootfs", + "The path on the uVM of the unpacked rootfs to use for the containers", + ) + flagSandboxPause = flag.Bool( + "pause-sandbox", + false, + "Use `/pause` as the sandbox container command", + ) +) + +var securityPolicy string + +func init() { + var err error + if securityPolicy, err = securitypolicy.NewOpenDoorPolicy().EncodeToString(); err != nil { + log.Fatal("could not encode open door policy to string: %w", err) + } +} + +func TestMain(m *testing.M) { + flag.Parse() + + if err := setup(); err != nil { + logrus.WithError(err).Fatal("could not set up testing") + } + + os.Exit(m.Run()) +} + +func setup() (err error) { + _ = os.MkdirAll(guestpath.LCOWRootPrefixInUVM, 0755) + + if vf := flag.Lookup("test.v"); vf != nil { + if vf.Value.String() == strconv.FormatBool(true) { + logrus.SetLevel(logrus.DebugLevel) + } else { + logrus.SetLevel(logrus.ErrorLevel) + } + } + + // should already start gcs cgroup + if !*flagJoinGCSCgroup { + gcsControl, err := cgroups.Load(cgroups.V1, cgroups.StaticPath("/")) + if err != nil { + return fmt.Errorf("failed to load root cgroup: %w", err) + } + if err := gcsControl.Add(cgroups.Process{Pid: os.Getpid()}); err != nil { + return fmt.Errorf("failed join root cgroup: %w", err) + } + logrus.Debug("joined root cgroup") + } + + // initialize runtime + rt, err := getRuntimeErr() + if err != nil { + return err + } + + // check policy will be parsed properly + if _, err = getHostErr(rt, getTransport()); err != nil { + return err + } + + return nil +} + +// +// host and runtime management +// + +func getTestState(ctx context.Context, t testing.TB) (*hcsv2.Host, runtime.Runtime) { + rt := getRuntime(ctx, t) + + return getHost(ctx, t, rt), rt +} + +func getHost(_ context.Context, t testing.TB, rt runtime.Runtime) *hcsv2.Host { + h, err := getHostErr(rt, getTransport()) + if err != nil { + t.Helper() + t.Fatalf("could not get host: %v", err) + } + + return h +} + +func getHostErr(rt runtime.Runtime, tp transport.Transport) (*hcsv2.Host, error) { + h := hcsv2.NewHost(rt, tp) + if err := h.SetConfidentialUVMOptions("", securityPolicy, ""); err != nil { + return nil, fmt.Errorf("could not set host security policy: %w", err) + } + + return h, nil +} + +func getRuntime(_ context.Context, t testing.TB) runtime.Runtime { + rt, err := getRuntimeErr() + if err != nil { + t.Helper() + t.Fatalf("could not get runtime: %v", err) + } + + return rt +} + +func getRuntimeErr() (runtime.Runtime, error) { + rt, err := runc.NewRuntime(guestpath.LCOWRootPrefixInUVM) + if err != nil { + return rt, fmt.Errorf("failed to initialize runc runtime: %w", err) + } + + return rt, nil +} + +func getTransport() transport.Transport { + return &PipeTransport{} +} + +func requireFeatures(t testing.TB, features ...string) { + require.Features(t, flagFeatures.S, features...) +} diff --git a/test/go.mod b/test/go.mod index 14bd43e43b..5e42bb99be 100644 --- a/test/go.mod +++ b/test/go.mod @@ -4,7 +4,8 @@ go 1.17 require ( github.com/Microsoft/go-winio v0.5.2 - github.com/Microsoft/hcsshim v0.9.3 + github.com/Microsoft/hcsshim v0.9.4 + github.com/containerd/cgroups v1.0.3 github.com/containerd/containerd v1.6.6 github.com/containerd/go-runc v1.0.0 github.com/containerd/ttrpc v1.1.0 @@ -29,18 +30,20 @@ require ( github.com/agnivade/levenshtein v1.0.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.1.3 // indirect - github.com/containerd/cgroups v1.0.3 // indirect github.com/containerd/console v1.0.3 // indirect github.com/containerd/continuity v0.2.2 // indirect github.com/containerd/fifo v1.0.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/cli v20.10.17+incompatible // indirect github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/docker v20.10.17+incompatible // indirect github.com/docker/docker-credential-helpers v0.6.4 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect + github.com/docker/go-units v0.4.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/gobwas/glob v0.2.3 // indirect + github.com/godbus/dbus/v5 v5.0.6 // indirect github.com/gogo/googleapis v1.4.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -49,9 +52,12 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/klauspost/compress v1.13.6 // indirect + github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3 // indirect + github.com/mattn/go-shellwords v1.0.12 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/sys/mountinfo v0.5.0 // indirect github.com/moby/sys/signal v0.6.0 // indirect + github.com/moby/sys/symlink v0.2.0 // indirect github.com/open-policy-agent/opa v0.42.2 // indirect github.com/opencontainers/runc v1.1.2 // indirect github.com/opencontainers/selinux v1.10.1 // indirect @@ -60,6 +66,8 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/vektah/gqlparser/v2 v2.4.5 // indirect + github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 // indirect + github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f // indirect diff --git a/test/go.sum b/test/go.sum index 39855f1af8..33a0520ffd 100644 --- a/test/go.sum +++ b/test/go.sum @@ -275,9 +275,11 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -330,6 +332,7 @@ github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -410,9 +413,11 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg78 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c h1:RBUpb2b14UnmRHNd2uHz20ZHLDK+SW5Us/vWF5IHRaY= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6 h1:mkgN1ofwASrYnJ5W6U/BxG15eXXXjirgZc7CLqkcaro= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= @@ -618,6 +623,7 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3 h1:jUp75lepDg0phMUJBCmvaeFDldD2N3S1lBuPwUTszio= github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -633,6 +639,7 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= @@ -664,6 +671,7 @@ github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdx github.com/moby/sys/signal v0.6.0 h1:aDpY94H8VlhTGa9sNYUFCFsMZIUh5wm0B6XkIoJj/iY= github.com/moby/sys/signal v0.6.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/sys/symlink v0.2.0 h1:tk1rOM+Ljp0nFmfOIBtlV3rTDlWOwFRhjEeAhZB0nZc= github.com/moby/sys/symlink v0.2.0/go.mod h1:7uZVF2dqJjG/NsClqul95CqKOBRQyYSNnJ6BMgR/gFs= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20210610120745-9d4ed1856297/go.mod h1:vgPCkQMyxTZ7IDy8SXRufE172gr8+K/JE/7hHFxHW3A= @@ -897,10 +905,12 @@ github.com/vektah/gqlparser/v2 v2.4.5/go.mod h1:flJWIR04IMQPGz+BXLrORkrARBxv/rty github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 h1:+UB2BJA852UkGH42H+Oee69djmxS3ANzl2b/JtT1YiA= github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= diff --git a/test/internal/oci/cri.go b/test/internal/oci/cri.go index 5157755cfb..99787e6be1 100644 --- a/test/internal/oci/cri.go +++ b/test/internal/oci/cri.go @@ -2,7 +2,7 @@ package oci import ( imagespec "github.com/opencontainers/image-spec/specs-go/v1" - runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2" + runtime "k8s.io/cri-api/pkg/apis/runtime/v1" ) //