From da627517a0d29efccf9e2011861e8c4b0d0248ec Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 10 Apr 2020 11:06:21 -0700 Subject: [PATCH 01/38] WIP --- eng/pipelines/coreclr/perf.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index ffa22b8afc7515..58600dd8a3b956 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -60,7 +60,16 @@ jobs: - Windows_NT_x86 jobParameters: testGroup: perf - + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/mono/templates/build-job.yml + runtimeFlavor: mono + buildConfig: release + platforms: + - Linux_x64 + - Windows_NT_x64 + - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml From 52f6d0e8b1c94c2e3cef48b4e7a4538d3dc2743a Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 10 Apr 2020 13:57:07 -0700 Subject: [PATCH 02/38] Add mono perf job --- eng/pipelines/coreclr/perf.yml | 12 +++++++++++- eng/pipelines/coreclr/templates/perf-job.yml | 4 +++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index 58600dd8a3b956..b019e1b97b10f9 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -68,7 +68,17 @@ jobs: buildConfig: release platforms: - Linux_x64 - - Windows_NT_x64 + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + runtimeFlavor: mono - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index d71e26001c44c1..5d1be33bc40ca8 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -43,7 +43,9 @@ jobs: - ${{ parameters.framework }} steps: # Extra steps that will be passed to the performance template and run before sending the job to helix (all of which is done in the template) - + - ${{ if eq(${{ variables.runtimeFlavorName }}, 'Mono')}}: + - script: "echo Hello" + displayName: "Test of mono" # Optionally download live-built libraries - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: From 11ec5afe9fc943f22d56092f5a5ed6db1b901d55 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 10 Apr 2020 14:01:35 -0700 Subject: [PATCH 03/38] Move runtimeFlavor to the right level --- eng/pipelines/coreclr/perf.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index b019e1b97b10f9..e3533cbb64a152 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -73,12 +73,12 @@ jobs: parameters: jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml buildConfig: release + runtimeFlavor: mono platforms: - Linux_x64 jobParameters: testGroup: perf liveLibrariesBuildConfig: Release - runtimeFlavor: mono - template: /eng/pipelines/common/platform-matrix.yml parameters: From 4e4ac5ae00eecd0750a09214d81bbce727693de2 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 10 Apr 2020 14:08:16 -0700 Subject: [PATCH 04/38] chnage variable access --- eng/pipelines/coreclr/templates/perf-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 5d1be33bc40ca8..07ec7db9afdf57 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -43,7 +43,7 @@ jobs: - ${{ parameters.framework }} steps: # Extra steps that will be passed to the performance template and run before sending the job to helix (all of which is done in the template) - - ${{ if eq(${{ variables.runtimeFlavorName }}, 'Mono')}}: + - ${{ if eq(variables.runtimeFlavorName, 'Mono')}}: - script: "echo Hello" displayName: "Test of mono" From ab9008e1a491670e3f7f41428b71b3791fd77ddb Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 10 Apr 2020 14:54:32 -0700 Subject: [PATCH 05/38] Change condition --- eng/pipelines/coreclr/templates/perf-job.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 07ec7db9afdf57..5d86677f280e86 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -43,9 +43,9 @@ jobs: - ${{ parameters.framework }} steps: # Extra steps that will be passed to the performance template and run before sending the job to helix (all of which is done in the template) - - ${{ if eq(variables.runtimeFlavorName, 'Mono')}}: - - script: "echo Hello" - displayName: "Test of mono" + - script: "echo Hello" + displayName: "Test of mono" + condition: and(succeeded(), eq(variables['runtimeFlavorName'], 'Mono')) # Optionally download live-built libraries - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: From 15a0eda0ee94120177007a89265f9ef6e0990173 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 10 Apr 2020 16:38:17 -0700 Subject: [PATCH 06/38] Add steps to download mono and build the patched dotnet --- eng/pipelines/coreclr/perf.yml | 1 + eng/pipelines/coreclr/templates/perf-job.yml | 22 +++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index e3533cbb64a152..367d2000821a4d 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -84,6 +84,7 @@ jobs: parameters: jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml buildConfig: release + runtimeFlavor: coreclr platforms: - Linux_x64 - Windows_NT_x64 diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 5d86677f280e86..107c57b3dd936b 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -18,8 +18,8 @@ jobs: - template: run-performance-job.yml parameters: # Compute job name from template parameters - jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Performance {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.variables['runtimeFlavorName']) }} + displayName: ${{ format('Performance {0}{1} {2} {3} {4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.variables['runtimeFlavorName']) }} pool: ${{ parameters.pool }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -31,6 +31,8 @@ jobs: - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - ${{ if eq(parameters.variables['runtimeFlavorName'], 'Mono') }}: + - ${{ format('mono_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ if eq(parameters.osGroup, 'Windows_NT') }}: extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }} @@ -43,9 +45,6 @@ jobs: - ${{ parameters.framework }} steps: # Extra steps that will be passed to the performance template and run before sending the job to helix (all of which is done in the template) - - script: "echo Hello" - displayName: "Test of mono" - condition: and(succeeded(), eq(variables['runtimeFlavorName'], 'Mono')) # Optionally download live-built libraries - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: @@ -66,7 +65,20 @@ jobs: artifactName: '$(buildProductArtifactName)' displayName: 'product build' + - ${{ if eq(variables['runtimeFlavorName'], 'Mono') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/mono/$(osGroup).$(archType).$(buildConfigUpper) + cleanUnpackFolder: false + artifactFileName: 'MonoProduct_$(osGroup)_$(archType)_$(buildConfig)$(archiveExtension)' + artifactName: 'MonoProduct_$(osGroup)_$(archType)_$(buildConfig)' + displayName: 'Mono runtime' # Create Core_Root - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) displayName: Create Core_Root + condition: and(succeeded(), ne(variables['runtimeFlavorName'], 'Mono')) + + - script: "cd src/mono/netcore;make patch-mono-dotnet" + displayName: "Create mono dotnet" + condition: and(succeeded(), eq(variables['runtimeFlavorName'], 'Mono')) From d03fead4ed4f7216c82c937dedb889891cd2d53a Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Mon, 13 Apr 2020 10:52:17 -0700 Subject: [PATCH 07/38] Change variable access --- eng/pipelines/coreclr/templates/perf-job.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 107c57b3dd936b..045a978863fb21 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -18,8 +18,8 @@ jobs: - template: run-performance-job.yml parameters: # Compute job name from template parameters - jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.variables['runtimeFlavorName']) }} - displayName: ${{ format('Performance {0}{1} {2} {3} {4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.variables['runtimeFlavorName']) }} + jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.variables.runtimeFlavorName) }} + displayName: ${{ format('Performance {0}{1} {2} {3} {4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.variables.runtimeFlavorName) }} pool: ${{ parameters.pool }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -65,7 +65,7 @@ jobs: artifactName: '$(buildProductArtifactName)' displayName: 'product build' - - ${{ if eq(variables['runtimeFlavorName'], 'Mono') }}: + - ${{ if eq(parameters.variables.runtimeFlavorName, 'Mono') }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/mono/$(osGroup).$(archType).$(buildConfigUpper) From f0a8eb6e74dc8ae034db30959922f90fef55b8c5 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Mon, 13 Apr 2020 13:31:02 -0700 Subject: [PATCH 08/38] Pass variables directly --- eng/pipelines/coreclr/perf.yml | 2 +- eng/pipelines/coreclr/templates/perf-job.yml | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index 367d2000821a4d..e60aa9429c14ba 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -73,12 +73,12 @@ jobs: parameters: jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml buildConfig: release - runtimeFlavor: mono platforms: - Linux_x64 jobParameters: testGroup: perf liveLibrariesBuildConfig: Release + runtimeType: mono - template: /eng/pipelines/common/platform-matrix.yml parameters: diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 045a978863fb21..1c14e4012840eb 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -7,6 +7,7 @@ parameters: framework: netcoreapp5.0 # Specify the appropriate framework when running release branches (ie netcoreapp3.0 for release/3.0) liveLibrariesBuildConfig: '' variables: {} + runtimeType: 'coreclr' pool: '' ### Perf job @@ -18,8 +19,8 @@ jobs: - template: run-performance-job.yml parameters: # Compute job name from template parameters - jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.variables.runtimeFlavorName) }} - displayName: ${{ format('Performance {0}{1} {2} {3} {4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.variables.runtimeFlavorName) }} + jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType) }} + displayName: ${{ format('Performance {0}{1} {2} {3} {4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType) }} pool: ${{ parameters.pool }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} @@ -31,7 +32,7 @@ jobs: - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} - - ${{ if eq(parameters.variables['runtimeFlavorName'], 'Mono') }}: + - ${{ if eq(parameters.runtimeType, 'mono') }}: - ${{ format('mono_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ if eq(parameters.osGroup, 'Windows_NT') }}: @@ -65,7 +66,7 @@ jobs: artifactName: '$(buildProductArtifactName)' displayName: 'product build' - - ${{ if eq(parameters.variables.runtimeFlavorName, 'Mono') }}: + - ${{ if eq(parameters.runtimeType, 'mono') }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/mono/$(osGroup).$(archType).$(buildConfigUpper) @@ -77,8 +78,8 @@ jobs: # Create Core_Root - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) displayName: Create Core_Root - condition: and(succeeded(), ne(variables['runtimeFlavorName'], 'Mono')) + condition: and(succeeded(), ne(parameters.runtimeFlavorName, 'mono')) - script: "cd src/mono/netcore;make patch-mono-dotnet" displayName: "Create mono dotnet" - condition: and(succeeded(), eq(variables['runtimeFlavorName'], 'Mono')) + condition: and(succeeded(), eq(parameters.runtimeType, 'mono')) From 623ec9b6ee513be01f81335512fe89e40d34d9e7 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Mon, 13 Apr 2020 14:12:21 -0700 Subject: [PATCH 09/38] Plumb through mono path to runner --- eng/common/performance/perfhelixpublish.proj | 12 ++++++++++-- eng/common/performance/performance-setup.ps1 | 9 +++++++++ eng/common/performance/performance-setup.sh | 13 +++++++++++++ eng/pipelines/coreclr/templates/perf-job.yml | 10 ++++++++-- 4 files changed, 40 insertions(+), 4 deletions(-) diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index cf5941e1b64554..b26901af23cd44 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -6,6 +6,7 @@ py -3 %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe + $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline @@ -40,6 +41,13 @@ $HELIX_WORKITEM_ROOT/testResults.xml + + --cli %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\dotnet + + + --cli $(BaseDirectory)/dotnet-mono/dotnet + + --corerun $(CoreRun) @@ -84,7 +92,7 @@ $(WorkItemDirectory) $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(MonoPath) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) 4:00 @@ -94,7 +102,7 @@ $(WorkItemDirectory) $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)" - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)" + $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(MonoPath) $(CoreRunArgument)" $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults) 4:00 diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 1763a1a97b0e79..971da6952e5db1 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -14,6 +14,7 @@ Param( [string] $Kind="micro", [switch] $Internal, [switch] $Compare, + [string] $MonoDotnet="", [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind" ) @@ -69,6 +70,13 @@ else { git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } +if($MonoDotnet -ne "") +{ + $UsingMono = "true" + $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") + Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath +} + if ($UseCoreRun) { $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot @@ -104,6 +112,7 @@ Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVar Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false # Helix Arguments Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index b9eecf94bd01b8..8eb82f367f2dad 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -12,6 +12,7 @@ commit_sha=$BUILD_SOURCEVERSION build_number=$BUILD_BUILDNUMBER internal=false compare=false +mono_dotnet= kind="micro" run_categories="Libraries Runtime" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" @@ -79,6 +80,10 @@ while (($# > 0)); do internal=true shift 1 ;; + --monodotnet) + mono=$2 + shift 2 + ;; --compare) compare=true shift 1 @@ -107,6 +112,7 @@ while (($# > 0)); do echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" echo " --internal If the benchmarks are running as an official job." + echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." echo "" exit 0 ;; @@ -191,6 +197,12 @@ if [[ "$use_core_run" = true ]]; then mv $core_root_directory $new_core_root fi +if [[ "$mono_dotnet" ~= "" ]]; then + using_mono = true + mono_dotnet_path=$payload_directory/dotnet-mono + mv $core_root_directory $mono_dotnet_path +fi + if [[ "$use_baseline_core_run" = true ]]; then new_baseline_core_root=$payload_directory/Baseline_Core_Root mv $baseline_core_root_directory $new_baseline_core_root @@ -221,3 +233,4 @@ Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false +Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 1c14e4012840eb..39963a232a3435 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -36,9 +36,15 @@ jobs: - ${{ format('mono_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }} + ${{ if eq(parameters.runtimeType, 'mono') }}: + extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }} -MonoDotnet $(Build.SourcesDirectory)\.dotnet-mono + ${{ if ne(parameters.runtimeType, 'mono') }}: + extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }} ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} + ${{ if eq(parameters.runtimeType, 'mono') }}: + extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono + ${{ if ne(parameters.runtimeType, 'mono') }}: + extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} variables: ${{ parameters.variables }} From edec8c8b2193ce7df3eb4c17a1749b487c804d96 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Mon, 13 Apr 2020 15:31:56 -0700 Subject: [PATCH 10/38] Fix parameter reference --- eng/pipelines/coreclr/templates/perf-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 39963a232a3435..1c0bcead13206b 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -84,7 +84,7 @@ jobs: # Create Core_Root - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) displayName: Create Core_Root - condition: and(succeeded(), ne(parameters.runtimeFlavorName, 'mono')) + condition: and(succeeded(), ne(parameters.runtimeType, 'mono')) - script: "cd src/mono/netcore;make patch-mono-dotnet" displayName: "Create mono dotnet" From e0d0defbdb91143086598bfa36f80405dc33de76 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Mon, 13 Apr 2020 16:50:21 -0700 Subject: [PATCH 11/38] Fix variables access? --- eng/common/performance/performance-setup.sh | 4 ++-- eng/pipelines/coreclr/perf.yml | 1 + eng/pipelines/coreclr/templates/perf-job.yml | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 8eb82f367f2dad..9cdd81ee8b6434 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -197,8 +197,8 @@ if [[ "$use_core_run" = true ]]; then mv $core_root_directory $new_core_root fi -if [[ "$mono_dotnet" ~= "" ]]; then - using_mono = true +if [[ "$mono_dotnet" != "" ]]; then + using_mono=true mono_dotnet_path=$payload_directory/dotnet-mono mv $core_root_directory $mono_dotnet_path fi diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index e60aa9429c14ba..952fab68278494 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -73,6 +73,7 @@ jobs: parameters: jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml buildConfig: release + runtimeFlavor: mono platforms: - Linux_x64 jobParameters: diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 1c0bcead13206b..3bf1a2709372c6 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -84,8 +84,8 @@ jobs: # Create Core_Root - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) displayName: Create Core_Root - condition: and(succeeded(), ne(parameters.runtimeType, 'mono')) + condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono')) - script: "cd src/mono/netcore;make patch-mono-dotnet" displayName: "Create mono dotnet" - condition: and(succeeded(), eq(parameters.runtimeType, 'mono')) + condition: and(succeeded(), (variables.runtimeFlavorName, 'Mono')) From 60f4dcfae86c2a13d67d1a2b3e09e1195b9ca6b9 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Mon, 13 Apr 2020 16:57:31 -0700 Subject: [PATCH 12/38] Fix typo --- eng/pipelines/coreclr/templates/perf-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 3bf1a2709372c6..a7af8575eec5e3 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -88,4 +88,4 @@ jobs: - script: "cd src/mono/netcore;make patch-mono-dotnet" displayName: "Create mono dotnet" - condition: and(succeeded(), (variables.runtimeFlavorName, 'Mono')) + condition: and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')) From 6f0928572770f7e8d0706d54a1f07d2d708e44ca Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Mon, 13 Apr 2020 17:56:26 -0700 Subject: [PATCH 13/38] Fix pathing issue --- eng/pipelines/coreclr/templates/perf-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index a7af8575eec5e3..e8a55d26c78e7b 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -75,7 +75,7 @@ jobs: - ${{ if eq(parameters.runtimeType, 'mono') }}: - template: /eng/pipelines/common/download-artifact-step.yml parameters: - unpackFolder: $(librariesDownloadDir)/mono/$(osGroup).$(archType).$(buildConfigUpper) + unpackFolder: $(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) cleanUnpackFolder: false artifactFileName: 'MonoProduct_$(osGroup)_$(archType)_$(buildConfig)$(archiveExtension)' artifactName: 'MonoProduct_$(osGroup)_$(archType)_$(buildConfig)' From f30649e7f1e6bfc7d33744ed4aa44b276439cc9b Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 14 Apr 2020 09:59:35 -0700 Subject: [PATCH 14/38] Add log saving for publish to helix step --- eng/common/performance/performance-setup.sh | 4 ++++ eng/pipelines/coreclr/templates/perf-job.yml | 4 ++-- eng/pipelines/coreclr/templates/run-performance-job.yml | 7 +++++++ 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 9cdd81ee8b6434..7779c3c1fdc25f 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -20,6 +20,7 @@ configurations="CompliationMode=$compilation_mode RunKind=$kind" run_from_perf_repo=false use_core_run=true use_baseline_core_run=true +using_mono=false while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -200,9 +201,12 @@ fi if [[ "$mono_dotnet" != "" ]]; then using_mono=true mono_dotnet_path=$payload_directory/dotnet-mono + echo $mono_dotnet_path mv $core_root_directory $mono_dotnet_path fi +echo $using_mono + if [[ "$use_baseline_core_run" = true ]]; then new_baseline_core_root=$payload_directory/Baseline_Core_Root mv $baseline_core_root_directory $new_baseline_core_root diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index e8a55d26c78e7b..b05f7cd45cfe23 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -37,12 +37,12 @@ jobs: ${{ if eq(parameters.osGroup, 'Windows_NT') }}: ${{ if eq(parameters.runtimeType, 'mono') }}: - extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }} -MonoDotnet $(Build.SourcesDirectory)\.dotnet-mono + extraSetupParameters: -Architecture ${{ parameters.archType }} -MonoDotnet $(Build.SourcesDirectory)\.dotnet-mono ${{ if ne(parameters.runtimeType, 'mono') }}: extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }} ${{ if ne(parameters.osGroup, 'Windows_NT') }}: ${{ if eq(parameters.runtimeType, 'mono') }}: - extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono + extraSetupParameters: --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono ${{ if ne(parameters.runtimeType, 'mono') }}: extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml index 1e37e86c8a6459..f7c2ef25462aef 100644 --- a/eng/pipelines/coreclr/templates/run-performance-job.yml +++ b/eng/pipelines/coreclr/templates/run-performance-job.yml @@ -103,3 +103,10 @@ jobs: WorkItemTimeout: 4:00 # 4 hours WorkItemDirectory: '$(WorkItemDirectory)' # WorkItemDirectory can not be empty, so we send it some docs to keep it happy CorrelationPayloadDirectory: '$(PayloadDirectory)' # it gets checked out to a folder with shorter path than WorkItemDirectory so we can avoid file name too long exceptions + - task: PublishPipelineArtifact@1 + displayName: Publish Logs + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ variables.runtimeFlavorName }}' + continueOnError: true + condition: always() From 94936e9ce3936c3a789a6387aa354e9ff2ff96e4 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 14 Apr 2020 10:53:57 -0700 Subject: [PATCH 15/38] Fix variable in shell script --- eng/common/performance/performance-setup.sh | 2 +- eng/pipelines/coreclr/templates/run-performance-job.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 7779c3c1fdc25f..8a63eb5f2e4c0d 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -82,7 +82,7 @@ while (($# > 0)); do shift 1 ;; --monodotnet) - mono=$2 + mono_dotnet=$2 shift 2 ;; --compare) diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml index f7c2ef25462aef..d9608a8cc6e9df 100644 --- a/eng/pipelines/coreclr/templates/run-performance-job.yml +++ b/eng/pipelines/coreclr/templates/run-performance-job.yml @@ -107,6 +107,6 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ variables.runtimeFlavorName }}' + artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_$(variables.runtimeFlavorName)' continueOnError: true condition: always() From f7f10432ddd2a66f27a1af5aefbc5b978a192d8b Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 14 Apr 2020 13:15:04 -0700 Subject: [PATCH 16/38] Fix up mv command and publish logs step --- eng/common/performance/performance-setup.sh | 5 +---- eng/pipelines/coreclr/templates/perf-job.yml | 1 + eng/pipelines/coreclr/templates/run-performance-job.yml | 3 ++- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 8a63eb5f2e4c0d..b7e6528135993d 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -201,12 +201,9 @@ fi if [[ "$mono_dotnet" != "" ]]; then using_mono=true mono_dotnet_path=$payload_directory/dotnet-mono - echo $mono_dotnet_path - mv $core_root_directory $mono_dotnet_path + mv $mono_dotnet $mono_dotnet_path fi -echo $using_mono - if [[ "$use_baseline_core_run" = true ]]; then new_baseline_core_root=$payload_directory/Baseline_Core_Root mv $baseline_core_root_directory $new_baseline_core_root diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index b05f7cd45cfe23..332df9fd64f271 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -27,6 +27,7 @@ jobs: osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup }} liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + runtimeType: ${{ parameters.runtimeType }} # Test job depends on the corresponding build job dependsOn: - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml index d9608a8cc6e9df..881a498fbd20de 100644 --- a/eng/pipelines/coreclr/templates/run-performance-job.yml +++ b/eng/pipelines/coreclr/templates/run-performance-job.yml @@ -16,6 +16,7 @@ parameters: timeoutInMinutes: 320 # optional -- timeout for the job enableTelemetry: false # optional -- enable for telemetry liveLibrariesBuildConfig: '' # optional -- live-live libraries configuration to use for the run + runtimeType: 'coreclr' jobs: - template: xplat-pipeline-job.yml @@ -107,6 +108,6 @@ jobs: displayName: Publish Logs inputs: targetPath: $(Build.SourcesDirectory)/artifacts/log - artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_$(variables.runtimeFlavorName)' + artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.runtimeType }}' continueOnError: true condition: always() From e90947b8585adf4477c52016c0323379a8ad0bd5 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 14 Apr 2020 14:41:29 -0700 Subject: [PATCH 17/38] Change runkind and add Windows runs --- eng/pipelines/coreclr/perf.yml | 2 ++ eng/pipelines/coreclr/templates/perf-job.yml | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index 952fab68278494..07a5d8b4bc7057 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -76,6 +76,8 @@ jobs: runtimeFlavor: mono platforms: - Linux_x64 + - Windows_NT_x64 + - Windows_NT_x86 jobParameters: testGroup: perf liveLibrariesBuildConfig: Release diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 332df9fd64f271..e66e955a3994c7 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -38,12 +38,12 @@ jobs: ${{ if eq(parameters.osGroup, 'Windows_NT') }}: ${{ if eq(parameters.runtimeType, 'mono') }}: - extraSetupParameters: -Architecture ${{ parameters.archType }} -MonoDotnet $(Build.SourcesDirectory)\.dotnet-mono + extraSetupParameters: -Architecture ${{ parameters.archType }} -MonoDotnet $(Build.SourcesDirectory)\.dotnet-mono -Kind micro_mono ${{ if ne(parameters.runtimeType, 'mono') }}: extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }} ${{ if ne(parameters.osGroup, 'Windows_NT') }}: ${{ if eq(parameters.runtimeType, 'mono') }}: - extraSetupParameters: --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono + extraSetupParameters: --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono --kind micro_mono ${{ if ne(parameters.runtimeType, 'mono') }}: extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} From 01dc157138b0fb0423ad769a4522f46bad6a134d Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 14 Apr 2020 14:55:27 -0700 Subject: [PATCH 18/38] Add Windows mono build --- eng/pipelines/coreclr/perf.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index 07a5d8b4bc7057..f29f71365cd723 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -68,6 +68,8 @@ jobs: buildConfig: release platforms: - Linux_x64 + - Windows_NT_x64 + - Windows_NT_x86 - template: /eng/pipelines/common/platform-matrix.yml parameters: From f959d0edbe23e29b92a1b567ac898a48871ebd4e Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 14 Apr 2020 17:24:52 -0700 Subject: [PATCH 19/38] Fixup parsing --- eng/common/performance/performance-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index b7e6528135993d..57618cdaa73910 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -67,6 +67,7 @@ while (($# > 0)); do ;; --kind) kind=$2 + configurations="CompliationMode=$compilation_mode RunKind=$kind" shift 2 ;; --runcategories) From 1becb9ee72ba0467ea741073ed76a258bb92ebd5 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Thu, 16 Apr 2020 12:21:16 -0700 Subject: [PATCH 20/38] Remove Windows runs for now --- eng/pipelines/coreclr/perf.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index f29f71365cd723..952fab68278494 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -68,8 +68,6 @@ jobs: buildConfig: release platforms: - Linux_x64 - - Windows_NT_x64 - - Windows_NT_x86 - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -78,8 +76,6 @@ jobs: runtimeFlavor: mono platforms: - Linux_x64 - - Windows_NT_x64 - - Windows_NT_x86 jobParameters: testGroup: perf liveLibrariesBuildConfig: Release From d69da0f3ee2431c6a0155a81b6e5218179750c73 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Mon, 20 Apr 2020 11:22:39 -0700 Subject: [PATCH 21/38] Switch to using build command to generate mono testhost --- eng/pipelines/coreclr/templates/perf-job.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index e66e955a3994c7..0f9c2310db3dce 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -87,6 +87,10 @@ jobs: displayName: Create Core_Root condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono')) - - script: "cd src/mono/netcore;make patch-mono-dotnet" - displayName: "Create mono dotnet" - condition: and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')) + - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)artifacts\\bin\\testhost\\$(buildConfig)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y" + displayName: "Create mono dotnet (Windows)" + condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono'), eq(variables.osGroup, 'Windows_NT'))) + + - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)artifacts/bin/testhost/$(buildConfig)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono" + displayName: "Create mono dotnet (Linux)" + condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono'), ne(variables.osGroup, 'Windows_NT'))) From 6ae7909bbaa49a35caccbb1bd44ca298812c0e0b Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Mon, 20 Apr 2020 11:33:16 -0700 Subject: [PATCH 22/38] Fixup parens --- eng/pipelines/coreclr/templates/perf-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 0f9c2310db3dce..c7e20acb024d79 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -89,8 +89,8 @@ jobs: - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)artifacts\\bin\\testhost\\$(buildConfig)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y" displayName: "Create mono dotnet (Windows)" - condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono'), eq(variables.osGroup, 'Windows_NT'))) + condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'Windows_NT')) - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)artifacts/bin/testhost/$(buildConfig)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono" displayName: "Create mono dotnet (Linux)" - condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono'), ne(variables.osGroup, 'Windows_NT'))) + condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'Windows_NT')) From 58f8fde4062bda79b607a445af98e4d7666c93da Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Mon, 20 Apr 2020 15:53:50 -0700 Subject: [PATCH 23/38] Forgot to add a slash --- eng/pipelines/coreclr/templates/perf-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index c7e20acb024d79..021b76a4d5804a 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -91,6 +91,6 @@ jobs: displayName: "Create mono dotnet (Windows)" condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'Windows_NT')) - - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)artifacts/bin/testhost/$(buildConfig)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono" + - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)/artifacts/bin/testhost/$(buildConfig)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono" displayName: "Create mono dotnet (Linux)" condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'Windows_NT')) From 6904d3333e206980d1d3e9b40cd6bdefbd7aaef8 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 21 Apr 2020 09:47:11 -0700 Subject: [PATCH 24/38] Add Windows runs and fix pathing issue --- eng/pipelines/coreclr/perf.yml | 4 ++++ eng/pipelines/coreclr/templates/perf-job.yml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index 952fab68278494..f29f71365cd723 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -68,6 +68,8 @@ jobs: buildConfig: release platforms: - Linux_x64 + - Windows_NT_x64 + - Windows_NT_x86 - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -76,6 +78,8 @@ jobs: runtimeFlavor: mono platforms: - Linux_x64 + - Windows_NT_x64 + - Windows_NT_x86 jobParameters: testGroup: perf liveLibrariesBuildConfig: Release diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 021b76a4d5804a..25eb576609b1ea 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -87,10 +87,10 @@ jobs: displayName: Create Core_Root condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono')) - - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)artifacts\\bin\\testhost\\$(buildConfig)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y" + - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y" displayName: "Create mono dotnet (Windows)" condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'Windows_NT')) - - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)/artifacts/bin/testhost/$(buildConfig)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono" + - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)/artifacts/bin/testhost/$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono" displayName: "Create mono dotnet (Linux)" condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'Windows_NT')) From 55f4acceb731437d550893bfa62a2eb9a7dad59b Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 21 Apr 2020 13:48:01 -0700 Subject: [PATCH 25/38] Add backslash for Windows and -r for cp on Linux --- eng/pipelines/coreclr/templates/perf-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 25eb576609b1ea..8178cdf0059940 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -87,10 +87,10 @@ jobs: displayName: Create Core_Root condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono')) - - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y" + - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y" displayName: "Create mono dotnet (Windows)" condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'Windows_NT')) - - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)/artifacts/bin/testhost/$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono" + - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)/artifacts/bin/testhost/$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono -r" displayName: "Create mono dotnet (Linux)" condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'Windows_NT')) From 08ffc75cd7dff87ca0fd1279e3377bea1399b478 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Wed, 22 Apr 2020 11:18:37 -0700 Subject: [PATCH 26/38] Switch to using corerun argument --- eng/common/performance/perfhelixpublish.proj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index b26901af23cd44..632b4b4dc394f5 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -42,10 +42,10 @@ - --cli %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\dotnet + --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\dotnet - --cli $(BaseDirectory)/dotnet-mono/dotnet + --corerun $(BaseDirectory)/dotnet-mono/dotnet From 163a1de2ab736dcac8109a0627861ae58b611194 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Wed, 22 Apr 2020 15:17:34 -0700 Subject: [PATCH 27/38] Switch to monopath --- eng/common/performance/perfhelixpublish.proj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index 632b4b4dc394f5..9e9a0c4bd123ef 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -42,10 +42,10 @@ - --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\dotnet + --monopath %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\dotnet - --corerun $(BaseDirectory)/dotnet-mono/dotnet + --monopath $(BaseDirectory)/dotnet-mono/dotnet From 7ca0b35d85c7244c62a847324dcbaf4e6fbd6c06 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Thu, 23 Apr 2020 11:55:32 -0700 Subject: [PATCH 28/38] Use CoreRun --- eng/common/performance/perfhelixpublish.proj | 4 ++-- eng/pipelines/coreclr/templates/perf-job.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index 9e9a0c4bd123ef..8cc77a0426c6ed 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -42,10 +42,10 @@ - --monopath %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\dotnet + --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe - --monopath $(BaseDirectory)/dotnet-mono/dotnet + --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 8178cdf0059940..414b4b3ee40eb0 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -87,10 +87,10 @@ jobs: displayName: Create Core_Root condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono')) - - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y" + - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\coreclr\\$(osGroup).$(archType).$(buildConfigUpper)\\corerun.exe $(Build.SourcesDirectory)\\.dotnet-mono\\shared\\Microsoft.NETCore.App\\5.0.0\\corerun.exe" displayName: "Create mono dotnet (Windows)" condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'Windows_NT')) - - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)/artifacts/bin/testhost/$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono -r" + - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)/artifacts/bin/testhost/$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono -r;cp $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)/corerun $(Build.SourcesDirectory)/.dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun" displayName: "Create mono dotnet (Linux)" condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'Windows_NT')) From 82fa61ebe4a001bcd76732595ed9cb01a4b96fc5 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Thu, 23 Apr 2020 13:20:17 -0700 Subject: [PATCH 29/38] Remove mono property and use corerun instead --- eng/common/performance/perfhelixpublish.proj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index 8cc77a0426c6ed..1c7fef34ed57dc 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -42,10 +42,10 @@ - --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe + --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe - --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun + --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun @@ -92,7 +92,7 @@ $(WorkItemDirectory) $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(MonoPath) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) 4:00 @@ -102,7 +102,7 @@ $(WorkItemDirectory) $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)" - $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(MonoPath) $(CoreRunArgument)" + $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)" $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults) 4:00 From 9a973a4db27b0e60c796c3772390f7df6858fa65 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Thu, 23 Apr 2020 15:04:53 -0700 Subject: [PATCH 30/38] Fixing closing tags --- eng/common/performance/perfhelixpublish.proj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index 1c7fef34ed57dc..1db5e8a84d758b 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -42,10 +42,10 @@ - --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe + --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe - --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun + --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun From edf722ff0fd4b2636a038d4f93e88d969b6bc193 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 24 Apr 2020 11:51:42 -0700 Subject: [PATCH 31/38] Change xcopy to copy --- eng/pipelines/coreclr/templates/perf-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index 414b4b3ee40eb0..72d95f5f146989 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -87,7 +87,7 @@ jobs: displayName: Create Core_Root condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono')) - - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\coreclr\\$(osGroup).$(archType).$(buildConfigUpper)\\corerun.exe $(Build.SourcesDirectory)\\.dotnet-mono\\shared\\Microsoft.NETCore.App\\5.0.0\\corerun.exe" + - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y;copy $(Build.SourcesDirectory)\\artifacts\\bin\\coreclr\\$(osGroup).$(archType).$(buildConfigUpper)\\corerun.exe $(Build.SourcesDirectory)\\.dotnet-mono\\shared\\Microsoft.NETCore.App\\5.0.0\\corerun.exe" displayName: "Create mono dotnet (Windows)" condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'Windows_NT')) From 1c5c87498097f77ef12e91c992b33ceea66dac94 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 24 Apr 2020 12:02:18 -0700 Subject: [PATCH 32/38] Add llvm config flag --- eng/common/performance/performance-setup.ps1 | 2 ++ eng/common/performance/performance-setup.sh | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 971da6952e5db1..8122700f891ba1 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -12,6 +12,7 @@ Param( [string] $RunCategories="Libraries Runtime", [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Kind="micro", + [switch] $LLVM, [switch] $Internal, [switch] $Compare, [string] $MonoDotnet="", @@ -75,6 +76,7 @@ if($MonoDotnet -ne "") $UsingMono = "true" $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath + $Configurations += " LLVM=$LLVM" } if ($UseCoreRun) { diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 57618cdaa73910..808368c1568e25 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -14,6 +14,7 @@ internal=false compare=false mono_dotnet= kind="micro" +llvm=false run_categories="Libraries Runtime" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" configurations="CompliationMode=$compilation_mode RunKind=$kind" @@ -82,6 +83,10 @@ while (($# > 0)); do internal=true shift 1 ;; + --llvm) + llvm=true + shift 1 + ;; --monodotnet) mono_dotnet=$2 shift 2 @@ -203,6 +208,7 @@ if [[ "$mono_dotnet" != "" ]]; then using_mono=true mono_dotnet_path=$payload_directory/dotnet-mono mv $mono_dotnet $mono_dotnet_path + $configurations="$configurations LLVM=$llvm" fi if [[ "$use_baseline_core_run" = true ]]; then From 19448eb7f010b325e9fa8b6a1553d16ba35a523c Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 24 Apr 2020 13:47:07 -0700 Subject: [PATCH 33/38] Add AOT and Interpreter config flags --- eng/common/performance/performance-setup.ps1 | 6 ++++-- eng/common/performance/performance-setup.sh | 12 +++++++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 8122700f891ba1..3688f685d49680 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -12,7 +12,9 @@ Param( [string] $RunCategories="Libraries Runtime", [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Kind="micro", - [switch] $LLVM, + [switch] $LLVM, + [switch] $MonoInterpreter, + [switch] $MonoAOT, [switch] $Internal, [switch] $Compare, [string] $MonoDotnet="", @@ -76,7 +78,7 @@ if($MonoDotnet -ne "") $UsingMono = "true" $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath - $Configurations += " LLVM=$LLVM" + $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" } if ($UseCoreRun) { diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 808368c1568e25..3a74615f985ecc 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -15,6 +15,8 @@ compare=false mono_dotnet= kind="micro" llvm=false +monointerpreter=false +monoaot=false run_categories="Libraries Runtime" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" configurations="CompliationMode=$compilation_mode RunKind=$kind" @@ -87,6 +89,14 @@ while (($# > 0)); do llvm=true shift 1 ;; + --monointerpreter) + monointerpreter=true + shift 1 + ;; + --monoaot) + monoaot=true + shift 1 + ;; --monodotnet) mono_dotnet=$2 shift 2 @@ -208,7 +218,7 @@ if [[ "$mono_dotnet" != "" ]]; then using_mono=true mono_dotnet_path=$payload_directory/dotnet-mono mv $mono_dotnet $mono_dotnet_path - $configurations="$configurations LLVM=$llvm" + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" fi if [[ "$use_baseline_core_run" = true ]]; then From 08af789fcd4437ed1cf9c352aa6125b0810523d0 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 24 Apr 2020 16:57:42 -0700 Subject: [PATCH 34/38] Change order to ensure config gets added --- eng/common/performance/performance-setup.ps1 | 16 ++++++++-------- eng/common/performance/performance-setup.sh | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 3688f685d49680..9ae5d714918172 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -53,6 +53,14 @@ if ($Internal) { $HelixSourcePrefix = "official" } +if($MonoDotnet -ne "") +{ + $UsingMono = "true" + $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") + Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath + $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" +} + # FIX ME: This is a workaround until we get this from the actual pipeline $CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" @@ -73,14 +81,6 @@ else { git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } -if($MonoDotnet -ne "") -{ - $UsingMono = "true" - $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") - Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath - $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" -} - if ($UseCoreRun) { $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 3a74615f985ecc..10589a6bdf31d2 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -187,6 +187,13 @@ if [[ "$internal" == true ]]; then fi fi +if [[ "$mono_dotnet" != "" ]]; then + using_mono=true + mono_dotnet_path=$payload_directory/dotnet-mono + mv $mono_dotnet $mono_dotnet_path + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" +fi + common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" @@ -214,13 +221,6 @@ if [[ "$use_core_run" = true ]]; then mv $core_root_directory $new_core_root fi -if [[ "$mono_dotnet" != "" ]]; then - using_mono=true - mono_dotnet_path=$payload_directory/dotnet-mono - mv $mono_dotnet $mono_dotnet_path - configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" -fi - if [[ "$use_baseline_core_run" = true ]]; then new_baseline_core_root=$payload_directory/Baseline_Core_Root mv $baseline_core_root_directory $new_baseline_core_root From 21ee8fad8fbcff0fe08126e0374bef53380db858 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Mon, 27 Apr 2020 10:06:26 -0700 Subject: [PATCH 35/38] Change order --- eng/common/performance/performance-setup.ps1 | 10 +++++++--- eng/common/performance/performance-setup.sh | 9 ++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 9ae5d714918172..f178e49c65831f 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -55,9 +55,6 @@ if ($Internal) { if($MonoDotnet -ne "") { - $UsingMono = "true" - $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") - Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" } @@ -81,6 +78,13 @@ else { git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } +if($MonoDotnet -ne "") +{ + $UsingMono = "true" + $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") + Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath +} + if ($UseCoreRun) { $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 10589a6bdf31d2..462fd59b02d2d9 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -188,9 +188,6 @@ if [[ "$internal" == true ]]; then fi if [[ "$mono_dotnet" != "" ]]; then - using_mono=true - mono_dotnet_path=$payload_directory/dotnet-mono - mv $mono_dotnet $mono_dotnet_path configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" fi @@ -216,6 +213,12 @@ else mv $docs_directory $workitem_directory fi +if [[ "$mono_dotnet" != "" ]]; then + using_mono=true + mono_dotnet_path=$payload_directory/dotnet-mono + mv $mono_dotnet $mono_dotnet_path +fi + if [[ "$use_core_run" = true ]]; then new_core_root=$payload_directory/Core_Root mv $core_root_directory $new_core_root From bb816d25d5f35ef21ce26a2f8219b5d3072bc96b Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Mon, 4 May 2020 16:11:14 -0700 Subject: [PATCH 36/38] Add exlusion filter for Perf_Image_Load tests --- eng/common/performance/performance-setup.ps1 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index f178e49c65831f..0de1d9f772c4b9 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -56,6 +56,16 @@ if ($Internal) { if($MonoDotnet -ne "") { $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" + if($ExtraBenchmarkDotNetArguments -eq "") + { + #FIX ME: We need to block these tests as they don't run on mono for now + $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image*" + } + else + { + #FIX ME: We need to block these tests as they don't run on mono for now + $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image*" + } } # FIX ME: This is a workaround until we get this from the actual pipeline From 7916bd3152aa0d4e1ade9561d2f5eb1797d613cd Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 5 May 2020 14:24:26 -0700 Subject: [PATCH 37/38] Add more to exclusion list --- eng/common/performance/performance-setup.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 0de1d9f772c4b9..6580cc0293cf6f 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -59,12 +59,12 @@ if($MonoDotnet -ne "") if($ExtraBenchmarkDotNetArguments -eq "") { #FIX ME: We need to block these tests as they don't run on mono for now - $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image*" + $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" } else { #FIX ME: We need to block these tests as they don't run on mono for now - $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image*" + $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" } } From b7e7a6c4ad6c7d77fddbfddf4f1ab6f480d9738d Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Wed, 6 May 2020 16:20:30 -0700 Subject: [PATCH 38/38] Remove Windows legs for now --- eng/pipelines/coreclr/perf.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index f29f71365cd723..952fab68278494 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -68,8 +68,6 @@ jobs: buildConfig: release platforms: - Linux_x64 - - Windows_NT_x64 - - Windows_NT_x86 - template: /eng/pipelines/common/platform-matrix.yml parameters: @@ -78,8 +76,6 @@ jobs: runtimeFlavor: mono platforms: - Linux_x64 - - Windows_NT_x64 - - Windows_NT_x86 jobParameters: testGroup: perf liveLibrariesBuildConfig: Release