Added LCOW functional tests and benchmarks for uVMs and containers.

scripts/Test-Functional.ps1

@@ -0,0 +1,62 @@
+# ex: .\scripts\Test-Functional.ps1 -Action Bench -Count 2 -BenchTime "2x"
+
+[CmdletBinding()]
+param (
+    [ValidateSet('Test', 'Bench', 'List')]
+    [alias('a')]
+    [string]
+    $Action = 'Bench',
+
+    [string]
+    $Note = '',
+
+    [string]
+    $OutDirectory = '.\test\results',
+
+    # test parameters
+    [int]
+    $Count = 1,
+
+    [string]
+    $BenchTime = '5s',
+
+    [string]
+    $Timeout = '10m',
+
+    [alias('tv')]
+    [switch]
+    $TestVerbose,
+
+    [string]
+    $Run = '',
+
+    [string]
+    $Feature = ''
+)
+
+Import-Module ( Join-Path $PSScriptRoot Testing.psm1 ) -Force
+
+$date = Get-Date
+$testcmd, $out = New-TestCommand `
+    -Action $Action `
+    -Path .\bin\test\functional.exe `
+    -Name functional `
+    -OutDirectory $OutDirectory `
+    -Date $date `
+    -Note $Note `
+    -TestVerbose:$TestVerbose `
+    -Count $Count `
+    -BenchTime $BenchTime `
+    -Timeout $Timeout `
+    -Run $Run `
+    -Feature $Feature `
+    -Verbose:$Verbose
+
+Invoke-TestCommand `
+    -TestCmd $testcmd `
+    -OutputFile $out `
+    -OutputCmd (&{ if ( $Action -eq 'Bench' ) { 'benchstat' } }) `
+    -Preamble `
+    -Date $Date `
+    -Note $Note `
+    -Verbose:$Verbose

scripts/Testing.psm1

@@ -0,0 +1,144 @@
+function New-TestCommand {
+    [CmdletBinding()]
+    param (
+        [ValidateSet('Test', 'Bench', 'List')]
+        [alias('a')]
+        [string]
+        $Action = 'Bench',
+
+        [Parameter(Mandatory)]
+        [string]
+        $Path,
+
+        [Parameter(Mandatory)]
+        [string]
+        $Name,
+
+        [Parameter(Mandatory)]
+        [string]
+        $OutDirectory ,
+
+        [DateTime]
+        $Date = (Get-Date),
+
+        [string]
+        $Note = '',
+
+        # test parameters
+        [alias('tv')]
+        [switch]
+        $TestVerbose = $false,
+
+        [int]
+        $Count = 1,
+
+        [string]
+        $BenchTime = '5s',
+
+        [string]
+        $Timeout = '10m',
+
+        [string]
+        $Run = '',
+
+        [string]
+        $Feature = ''
+    )
+
+    $OutDirectory = Resolve-Path $OutDirectory
+    Write-Verbose "creating $OutDirectory"
+
+    New-Item -ItemType 'directory' -Path $OutDirectory -Force > $null
+
+    $testcmd = "$Path `'-test.timeout=$Timeout`' `'-test.shuffle=on`' `'-test.count=$Count`' "
+
+    if ( $TestVerbose ) {
+        $testcmd += ' ''-test.v'' '
+    }
+
+    switch ( $Action ) {
+        'List' {
+            if ( $Run -eq '' ) {
+                $Run = '.'
+            }
+            $testcmd += " `'-test.list=$Run`' "
+        }
+        'Test' {
+            if ( $Run -ne '' ) {
+                $testcmd += " `'-test.run=$Run`' "
+            }
+        }
+        'Bench' {
+            if ( $Run -eq '' ) {
+                $Run = '.'
+            }
+            $testcmd += ' ''-test.run=^#'' ''-test.benchmem'' ' + `
+                " `'-test.bench=$Run`' `'-test.benchtime=$BenchTime`' "
+        }
+    }
+
+    if ( $Feature -ne '' ) {
+        $testcmd += " `'-feature=$Feature`' "
+    }
+
+    $f = $Name + '-' + $Action
+    if ($Note -ne '' ) {
+        $f += '-' + $Note
+    }
+    $out = Join-Path $OutDirectory "$f-$(Get-Date -Date $date -Format FileDateTime).txt"
+
+    return $testcmd, $out
+}
+
+function Invoke-TestCommand {
+    [CmdletBinding()]
+    param (
+        [Parameter(Mandatory)]
+        [string]
+        $TestCmd,
+
+        [string]
+        $TestCmdPreamble = $TestCmd,
+
+        [string]
+        $OutputFile = 'nul',
+
+        [string]
+        $OutputCmd,
+
+        [switch]
+        $Preamble,
+
+        [DateTime]
+        $Date = (Get-Date),
+
+        [string]
+        $Note
+    )
+
+    if ($OutputFile -eq '' ) {
+        $OutputFile = 'nul'
+    }
+
+    Write-Verbose "Saving output to: $OutputFile"
+    if ( $Preamble ) {
+        & {
+            Write-Output "test.date: $(Get-Date -Date $Date -UFormat '%FT%R%Z' -AsUTC)"
+            if ( $Note -ne '' ) {
+                Write-Output "note: $Note"
+            }
+            Write-Output "test.command: $TestCmdPreamble"
+            Write-Output "pkg.commit: $(git rev-parse HEAD)"
+        } | Tee-Object -Append -FilePath $OutputFile
+    }
+
+    Write-Verbose "Running command: $TestCmd"
+    Invoke-Expression $TestCmd | Tee-Object -Append -FilePath $OutputFile
+
+    if ( $OutputCmd -ne '' -and $OutputFile -ne 'nul' ) {
+        $oc = "$OutputCmd $OutputFile"
+        Write-Verbose "Running command: $oc"
+        Invoke-Expression $oc
+    }
+
+}

test/cri-containerd/clone_test.go

@@ -14,7 +14,7 @@ import (
 
 	"github.com/Microsoft/hcsshim/osversion"
 	"github.com/Microsoft/hcsshim/pkg/annotations"
-	testutilities "github.com/Microsoft/hcsshim/test/functional/utilities"
+	"github.com/Microsoft/hcsshim/test/internal/require"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 )
 
@@ -240,7 +240,7 @@ func cleanupContainer(t *testing.T, client runtime.RuntimeServiceClient, ctx con
 // cloned container from that template.
 func Test_CloneContainer_WCOW(t *testing.T) {
 	requireFeatures(t, featureWCOWHypervisor)
-	testutilities.RequiresBuild(t, osversion.V20H2)
+	require.Build(t, osversion.V20H2)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -265,7 +265,7 @@ func Test_CloneContainer_WCOW(t *testing.T) {
 // A test for creating multiple clones(3 clones) from one template container.
 func Test_MultiplClonedContainers_WCOW(t *testing.T) {
 	requireFeatures(t, featureWCOWHypervisor)
-	testutilities.RequiresBuild(t, osversion.V20H2)
+	require.Build(t, osversion.V20H2)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -302,7 +302,7 @@ func Test_MultiplClonedContainers_WCOW(t *testing.T) {
 // container.
 func Test_NormalContainerInClonedPod_WCOW(t *testing.T) {
 	requireFeatures(t, featureWCOWHypervisor)
-	testutilities.RequiresBuild(t, osversion.V20H2)
+	require.Build(t, osversion.V20H2)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -342,7 +342,7 @@ func Test_NormalContainerInClonedPod_WCOW(t *testing.T) {
 // of those pods.
 func Test_CloneContainersWithClonedPodPool_WCOW(t *testing.T) {
 	requireFeatures(t, featureWCOWHypervisor)
-	testutilities.RequiresBuild(t, osversion.V20H2)
+	require.Build(t, osversion.V20H2)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -390,7 +390,7 @@ func Test_CloneContainersWithClonedPodPool_WCOW(t *testing.T) {
 
 func Test_ClonedContainerRunningAfterDeletingTemplate(t *testing.T) {
 	requireFeatures(t, featureWCOWHypervisor)
-	testutilities.RequiresBuild(t, osversion.V20H2)
+	require.Build(t, osversion.V20H2)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -424,7 +424,7 @@ func Test_ClonedContainerRunningAfterDeletingTemplate(t *testing.T) {
 // can be made from each of them simultaneously.
 func Test_MultipleTemplateAndClones_WCOW(t *testing.T) {
 	requireFeatures(t, featureWCOWHypervisor)
-	testutilities.RequiresBuild(t, osversion.V20H2)
+	require.Build(t, osversion.V20H2)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -467,7 +467,7 @@ func Test_MultipleTemplateAndClones_WCOW(t *testing.T) {
 // and verifies that the request correctly fails with an error.
 func Test_VerifyCloneAndTemplateConfig(t *testing.T) {
 	requireFeatures(t, featureWCOWHypervisor)
-	testutilities.RequiresBuild(t, osversion.V20H2)
+	require.Build(t, osversion.V20H2)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

test/cri-containerd/container_downlevel_test.go

@@ -7,13 +7,13 @@ import (
 	"testing"
 
 	"github.com/Microsoft/hcsshim/osversion"
-	testutilities "github.com/Microsoft/hcsshim/test/functional/utilities"
+	"github.com/Microsoft/hcsshim/test/internal/require"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 )
 
 func Test_CreateContainer_DownLevel_WCOW_Hypervisor(t *testing.T) {
 	requireFeatures(t, featureWCOWHypervisor)
-	testutilities.RequiresBuild(t, osversion.V19H1)
+	require.Build(t, osversion.V19H1)
 
 	pullRequiredImages(t, []string{imageWindowsNanoserver17763})
 

test/cri-containerd/container_layers_packing_test.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/Microsoft/hcsshim/osversion"
 	"github.com/Microsoft/hcsshim/pkg/annotations"
-	testutilities "github.com/Microsoft/hcsshim/test/functional/utilities"
+	"github.com/Microsoft/hcsshim/test/internal/require"
 )
 
 const (
@@ -38,7 +38,7 @@ func validateTargets(ctx context.Context, t *testing.T, deviceNumber int, podID
 }
 
 func Test_Container_Layer_Packing_On_VPMem(t *testing.T) {
-	testutilities.RequiresBuild(t, osversion.V19H1)
+	require.Build(t, osversion.V19H1)
 
 	client := newTestRuntimeClient(t)
 	ctx, cancel := context.WithCancel(context.Background())
@@ -93,7 +93,7 @@ func Test_Container_Layer_Packing_On_VPMem(t *testing.T) {
 }
 
 func Test_Many_Container_Layers_Supported_On_VPMem(t *testing.T) {
-	testutilities.RequiresBuild(t, osversion.V19H1)
+	require.Build(t, osversion.V19H1)
 
 	client := newTestRuntimeClient(t)
 	ctx, cancel := context.WithCancel(context.Background())
@@ -124,7 +124,7 @@ func Test_Many_Container_Layers_Supported_On_VPMem(t *testing.T) {
 }
 
 func Test_Annotation_Disable_Multi_Mapping(t *testing.T) {
-	testutilities.RequiresBuild(t, osversion.V19H1)
+	require.Build(t, osversion.V19H1)
 
 	client := newTestRuntimeClient(t)
 	ctx, cancel := context.WithCancel(context.Background())

test/cri-containerd/container_update_test.go

@@ -11,7 +11,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/memory"
 	"github.com/Microsoft/hcsshim/osversion"
 	"github.com/Microsoft/hcsshim/pkg/annotations"
-	testutilities "github.com/Microsoft/hcsshim/test/functional/utilities"
+	"github.com/Microsoft/hcsshim/test/internal/require"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 )
 
@@ -33,7 +33,7 @@ func calculateJobCPURate(hostProcs uint32, processorCount uint32) uint32 {
 }
 
 func Test_Container_UpdateResources_CPUShare(t *testing.T) {
-	testutilities.RequiresBuild(t, osversion.V20H2)
+	require.Build(t, osversion.V20H2)
 	type config struct {
 		name             string
 		requiredFeatures []string

test/cri-containerd/createcontainer_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/memory"
 	"github.com/Microsoft/hcsshim/osversion"
 	"github.com/Microsoft/hcsshim/pkg/annotations"
-	testutilities "github.com/Microsoft/hcsshim/test/functional/utilities"
+	"github.com/Microsoft/hcsshim/test/internal/require"
 	runtime "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
 )
 
@@ -844,7 +844,7 @@ func Test_CreateContainer_CPUShares_LCOW(t *testing.T) {
 
 func Test_CreateContainer_Mount_File_LCOW(t *testing.T) {
 	requireFeatures(t, featureLCOW)
-	testutilities.RequiresBuild(t, osversion.V19H1)
+	require.Build(t, osversion.V19H1)
 
 	pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine})
 
@@ -889,7 +889,7 @@ func Test_CreateContainer_Mount_File_LCOW(t *testing.T) {
 
 func Test_CreateContainer_Mount_ReadOnlyFile_LCOW(t *testing.T) {
 	requireFeatures(t, featureLCOW)
-	testutilities.RequiresBuild(t, osversion.V19H1)
+	require.Build(t, osversion.V19H1)
 
 	pullRequiredLCOWImages(t, []string{imageLcowK8sPause, imageLcowAlpine})