diff --git a/eng/common/performance/perfhelixpublish.proj b/eng/common/performance/perfhelixpublish.proj index cf5941e1b64554..1db5e8a84d758b 100644 --- a/eng/common/performance/perfhelixpublish.proj +++ b/eng/common/performance/perfhelixpublish.proj @@ -6,6 +6,7 @@ py -3 %HELIX_CORRELATION_PAYLOAD%\Core_Root\CoreRun.exe %HELIX_CORRELATION_PAYLOAD%\Baseline_Core_Root\CoreRun.exe + $(HelixPreCommands);call %HELIX_CORRELATION_PAYLOAD%\performance\tools\machine-setup.cmd;set PYTHONPATH=%HELIX_WORKITEM_PAYLOAD%\scripts%3B%HELIX_WORKITEM_PAYLOAD% %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts %HELIX_CORRELATION_PAYLOAD%\artifacts\BenchmarkDotNet.Artifacts_Baseline @@ -40,6 +41,13 @@ $HELIX_WORKITEM_ROOT/testResults.xml + + --corerun %HELIX_CORRELATION_PAYLOAD%\dotnet-mono\shared\Microsoft.NETCore.App\5.0.0\corerun.exe + + + --corerun $(BaseDirectory)/dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun + + --corerun $(CoreRun) diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 1763a1a97b0e79..6580cc0293cf6f 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -12,8 +12,12 @@ Param( [string] $RunCategories="Libraries Runtime", [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Kind="micro", + [switch] $LLVM, + [switch] $MonoInterpreter, + [switch] $MonoAOT, [switch] $Internal, [switch] $Compare, + [string] $MonoDotnet="", [string] $Configurations="CompilationMode=$CompilationMode RunKind=$Kind" ) @@ -49,6 +53,21 @@ if ($Internal) { $HelixSourcePrefix = "official" } +if($MonoDotnet -ne "") +{ + $Configurations += " LLVM=$LLVM MonoInterpreter=$MonoInterpreter MonoAOT=$MonoAOT" + if($ExtraBenchmarkDotNetArguments -eq "") + { + #FIX ME: We need to block these tests as they don't run on mono for now + $ExtraBenchmarkDotNetArguments = "--exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" + } + else + { + #FIX ME: We need to block these tests as they don't run on mono for now + $ExtraBenchmarkDotNetArguments += " --exclusion-filter *Perf_Image* *Perf_NamedPipeStream*" + } +} + # FIX ME: This is a workaround until we get this from the actual pipeline $CommonSetupArguments="--channel master --queue $Queue --build-number $BuildNumber --build-configs $Configurations --architecture $Architecture" $SetupArguments = "--repository https://github.com/$Repository --branch $Branch --get-perf-hash --commit-sha $CommitSha $CommonSetupArguments" @@ -69,6 +88,13 @@ else { git clone --branch master --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } +if($MonoDotnet -ne "") +{ + $UsingMono = "true" + $MonoDotnetPath = (Join-Path $PayloadDirectory "dotnet-mono") + Move-Item -Path $MonoDotnet -Destination $MonoDotnetPath +} + if ($UseCoreRun) { $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot @@ -104,6 +130,7 @@ Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVar Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'MonoDotnet' -Value "$UsingMono" -IsMultiJobVariable $false # Helix Arguments Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index b9eecf94bd01b8..462fd59b02d2d9 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -12,13 +12,18 @@ commit_sha=$BUILD_SOURCEVERSION build_number=$BUILD_BUILDNUMBER internal=false compare=false +mono_dotnet= kind="micro" +llvm=false +monointerpreter=false +monoaot=false run_categories="Libraries Runtime" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" configurations="CompliationMode=$compilation_mode RunKind=$kind" run_from_perf_repo=false use_core_run=true use_baseline_core_run=true +using_mono=false while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -65,6 +70,7 @@ while (($# > 0)); do ;; --kind) kind=$2 + configurations="CompliationMode=$compilation_mode RunKind=$kind" shift 2 ;; --runcategories) @@ -79,6 +85,22 @@ while (($# > 0)); do internal=true shift 1 ;; + --llvm) + llvm=true + shift 1 + ;; + --monointerpreter) + monointerpreter=true + shift 1 + ;; + --monoaot) + monoaot=true + shift 1 + ;; + --monodotnet) + mono_dotnet=$2 + shift 2 + ;; --compare) compare=true shift 1 @@ -107,6 +129,7 @@ while (($# > 0)); do echo " --kind Related to csproj. The kind of benchmarks that should be run. Defaults to micro" echo " --runcategories Related to csproj. Categories of benchmarks to run. Defaults to \"coreclr corefx\"" echo " --internal If the benchmarks are running as an official job." + echo " --monodotnet Pass the path to the mono dotnet for mono performance testing." echo "" exit 0 ;; @@ -164,6 +187,10 @@ if [[ "$internal" == true ]]; then fi fi +if [[ "$mono_dotnet" != "" ]]; then + configurations="$configurations LLVM=$llvm MonoInterpreter=$monointerpreter MonoAOT=$monoaot" +fi + common_setup_arguments="--channel master --queue $queue --build-number $build_number --build-configs $configurations --architecture $architecture" setup_arguments="--repository https://github.com/$repository --branch $branch --get-perf-hash --commit-sha $commit_sha $common_setup_arguments" @@ -186,6 +213,12 @@ else mv $docs_directory $workitem_directory fi +if [[ "$mono_dotnet" != "" ]]; then + using_mono=true + mono_dotnet_path=$payload_directory/dotnet-mono + mv $mono_dotnet $mono_dotnet_path +fi + if [[ "$use_core_run" = true ]]; then new_core_root=$payload_directory/Core_Root mv $core_root_directory $new_core_root @@ -221,3 +254,4 @@ Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false +Write-PipelineSetVariable -name "MonoDotnet" -value "$using_mono" -is_multi_job_variable false diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index ffa22b8afc7515..952fab68278494 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -60,11 +60,32 @@ jobs: - Windows_NT_x86 jobParameters: testGroup: perf - + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/mono/templates/build-job.yml + runtimeFlavor: mono + buildConfig: release + platforms: + - Linux_x64 + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + runtimeType: mono + - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml buildConfig: release + runtimeFlavor: coreclr platforms: - Linux_x64 - Windows_NT_x64 diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index d71e26001c44c1..72d95f5f146989 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -7,6 +7,7 @@ parameters: framework: netcoreapp5.0 # Specify the appropriate framework when running release branches (ie netcoreapp3.0 for release/3.0) liveLibrariesBuildConfig: '' variables: {} + runtimeType: 'coreclr' pool: '' ### Perf job @@ -18,24 +19,33 @@ jobs: - template: run-performance-job.yml parameters: # Compute job name from template parameters - jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - displayName: ${{ format('Performance {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + jobName: ${{ format('perfbuild_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType) }} + displayName: ${{ format('Performance {0}{1} {2} {3} {4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.runtimeType) }} pool: ${{ parameters.pool }} buildConfig: ${{ parameters.buildConfig }} archType: ${{ parameters.archType }} osGroup: ${{ parameters.osGroup }} osSubgroup: ${{ parameters.osSubgroup }} liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + runtimeType: ${{ parameters.runtimeType }} # Test job depends on the corresponding build job dependsOn: - ${{ format('coreclr_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + - ${{ if eq(parameters.runtimeType, 'mono') }}: + - ${{ format('mono_product_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} ${{ if eq(parameters.osGroup, 'Windows_NT') }}: - extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }} + ${{ if eq(parameters.runtimeType, 'mono') }}: + extraSetupParameters: -Architecture ${{ parameters.archType }} -MonoDotnet $(Build.SourcesDirectory)\.dotnet-mono -Kind micro_mono + ${{ if ne(parameters.runtimeType, 'mono') }}: + extraSetupParameters: -CoreRootDirectory $(Build.SourcesDirectory)\artifacts\tests\coreclr\${{ parameters.osGroup }}.${{ parameters.archType }}.Release\Tests\Core_Root -Architecture ${{ parameters.archType }} ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} + ${{ if eq(parameters.runtimeType, 'mono') }}: + extraSetupParameters: --architecture ${{ parameters.archType }} --monodotnet $(Build.SourcesDirectory)/.dotnet-mono --kind micro_mono + ${{ if ne(parameters.runtimeType, 'mono') }}: + extraSetupParameters: --corerootdirectory $(Build.SourcesDirectory)/artifacts/tests/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.Release/Tests/Core_Root --architecture ${{ parameters.archType }} variables: ${{ parameters.variables }} @@ -44,7 +54,6 @@ jobs: steps: # Extra steps that will be passed to the performance template and run before sending the job to helix (all of which is done in the template) - # Optionally download live-built libraries - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - template: /eng/pipelines/common/download-artifact-step.yml @@ -64,7 +73,24 @@ jobs: artifactName: '$(buildProductArtifactName)' displayName: 'product build' + - ${{ if eq(parameters.runtimeType, 'mono') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) + cleanUnpackFolder: false + artifactFileName: 'MonoProduct_$(osGroup)_$(archType)_$(buildConfig)$(archiveExtension)' + artifactName: 'MonoProduct_$(osGroup)_$(archType)_$(buildConfig)' + displayName: 'Mono runtime' # Create Core_Root - script: $(coreClrRepoRootDir)build-test$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) displayName: Create Core_Root + condition: and(succeeded(), ne(variables.runtimeFlavorName, 'Mono')) + + - script: "build.cmd -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)\\bin\\mono\\$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;xcopy $(Build.SourcesDirectory)\\artifacts\\bin\\testhost\\$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)\\* $(Build.SourcesDirectory)\\.dotnet-mono /E /I /Y;copy $(Build.SourcesDirectory)\\artifacts\\bin\\coreclr\\$(osGroup).$(archType).$(buildConfigUpper)\\corerun.exe $(Build.SourcesDirectory)\\.dotnet-mono\\shared\\Microsoft.NETCore.App\\5.0.0\\corerun.exe" + displayName: "Create mono dotnet (Windows)" + condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), eq(variables.osGroup, 'Windows_NT')) + + - script: "mkdir $(Build.SourcesDirectory)/.dotnet-mono;./build.sh -subset libs.pretest -configuration release -ci -arch $(archType) -testscope innerloop /p:RuntimeArtifactsPath=$(librariesDownloadDir)/bin/mono/$(osGroup).$(archType).$(buildConfigUpper) /p:RuntimeFlavor=mono;cp $(Build.SourcesDirectory)/artifacts/bin/testhost/$(_Framework)-$(osGroup)-$(buildConfigUpper)-$(archType)/* $(Build.SourcesDirectory)/.dotnet-mono -r;cp $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(buildConfigUpper)/corerun $(Build.SourcesDirectory)/.dotnet-mono/shared/Microsoft.NETCore.App/5.0.0/corerun" + displayName: "Create mono dotnet (Linux)" + condition: and(and(succeeded(), eq(variables.runtimeFlavorName, 'Mono')), ne(variables.osGroup, 'Windows_NT')) diff --git a/eng/pipelines/coreclr/templates/run-performance-job.yml b/eng/pipelines/coreclr/templates/run-performance-job.yml index 1e37e86c8a6459..881a498fbd20de 100644 --- a/eng/pipelines/coreclr/templates/run-performance-job.yml +++ b/eng/pipelines/coreclr/templates/run-performance-job.yml @@ -16,6 +16,7 @@ parameters: timeoutInMinutes: 320 # optional -- timeout for the job enableTelemetry: false # optional -- enable for telemetry liveLibrariesBuildConfig: '' # optional -- live-live libraries configuration to use for the run + runtimeType: 'coreclr' jobs: - template: xplat-pipeline-job.yml @@ -103,3 +104,10 @@ jobs: WorkItemTimeout: 4:00 # 4 hours WorkItemDirectory: '$(WorkItemDirectory)' # WorkItemDirectory can not be empty, so we send it some docs to keep it happy CorrelationPayloadDirectory: '$(PayloadDirectory)' # it gets checked out to a folder with shorter path than WorkItemDirectory so we can avoid file name too long exceptions + - task: PublishPipelineArtifact@1 + displayName: Publish Logs + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: 'Performance_Run_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)_${{ parameters.runtimeType }}' + continueOnError: true + condition: always()