diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a91b0be8abe..f485b828b5a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,6 +8,9 @@ parameters: - name: runScheduledPrivateJobs type: boolean default: false +- name: onlySanityCheck + type: boolean + default: false resources: containers: @@ -52,724 +55,41 @@ schedules: always: true jobs: - -###################################################### -# Public correctness jobs -###################################################### - -- ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: - - # Scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: scenarios - projectFile: scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 9.0 - - 8.0 - - ## MAUI scenario benchmarks - #- template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/scenarios.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # isPublic: true - # jobParameters: - # kind: maui_scenarios - # projectFile: maui_scenarios.proj - # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - # - main - # - 8.0 - - # Blazor scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: blazor_scenarios - projectFile: blazor_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 9.0 - - 8.0 - - # SDK scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - win-x86 - #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available - isPublic: true - jobParameters: - kind: sdk_scenarios - projectFile: sdk_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 9.0 - - 8.0 - - # micro benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - win-x86 - isPublic: true - jobParameters: - kind: micro - csproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - main - - 9.0 - - 8.0 - - # Ubuntux64 Default and NativeAOT micro benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - ubuntu-x64 - isPublic: true - jobParameters: - kind: micro - csproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - main - - nativeaot9.0 - - nativeaot8.0 - - 9.0 - - 8.0 - - # net462 micro benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-rs5-x64 - isPublic: true - jobParameters: - kind: micro_net462 - csproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - net462 - - # ML.NET benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: mlnet - csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj - runCategories: 'mldotnet' - channels: - - main - - 9.0 - - 8.0 - - # F# benchmarks - - ${{ if false }}: # skipping, no useful benchmarks there currently - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: fsharp - csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj - runCategories: 'fsharp' - channels: - - main - - 9.0 - - 8.0 - - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: fsharpmicro - csproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj - runCategories: 'FSharpMicro' - channels: - - main - - 9.0 - - 8.0 - - # bepuphysics benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: bepuphysics - csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj - runCategories: 'BepuPhysics' - channels: - - main - - 9.0 - - 8.0 - - # ImageSharp benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: imagesharp - csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj - runCategories: 'ImageSharp' - channels: - - main - - 9.0 - - 8.0 - - # Akade.IndexedSet benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: akadeindexedset - csproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj - runCategories: 'AkadeIndexedSet' - channels: - - main - - 9.0 - - 8.0 - - # Roslyn benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: roslyn - csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - runCategories: 'roslyn' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - 9.0 - - 8.0 - - # ILLink benchmarks - # disabled because of: https://github.com/dotnet/performance/issues/3569 - # - template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/benchmark_jobs.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # isPublic: true - # jobParameters: - # kind: illink - # csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj - # runCategories: 'illink' - # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - # - main - - # NativeAOT scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: nativeaot_scenarios - projectFile: nativeaot_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 9.0 - - 8.0 - - # Powershell benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - kind: powershell - csproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj - runCategories: 'Public' - channels: - - main - - 9.0 - - 8.0 - -########################################### -# Private Jobs -########################################### - -- ${{ if or(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: - - # Scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: scenarios - projectFile: scenarios.proj - channels: - - main - - 9.0 - - 8.0 - - # Affinitized Scenario benchmarks (Initially just PDN) - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - win-arm64 - - win-arm64-ampere - isPublic: false - jobParameters: - kind: scenarios - projectFile: scenarios_affinitized.proj - channels: - - main - - 9.0 - - 8.0 - additionalJobIdentifier: 'Affinity_85' - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - - # Maui Android scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - kind: maui_scenarios_android - projectFile: maui_scenarios_android.proj - dotnetVersionsLinks: - 9.0: ./eng/Version.Details.xml - channels: - - 8.0 - - # Maui iOS Mono scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - kind: maui_scenarios_ios - projectFile: maui_scenarios_ios.proj - dotnetVersionsLinks: - 9.0: ./eng/Version.Details.xml - channels: - - 8.0 - runtimeFlavor: mono - - # Maui iOS Native AOT scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - kind: maui_scenarios_ios - projectFile: maui_scenarios_ios.proj - dotnetVersionsLinks: - 9.0: ./eng/Version.Details.xml - channels: - - 8.0 - runtimeFlavor: coreclr - - ## Maui scenario benchmarks - #- template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/scenarios.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # - win-arm64 - # - ubuntu-arm64-ampere - # isPublic: false - # jobParameters: - # kind: maui_scenarios - # projectFile: maui_scenarios.proj - # channels: - # - main - # - 8.0 - - # NativeAOT scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - isPublic: false - jobParameters: - kind: nativeaot_scenarios - projectFile: nativeaot_scenarios.proj - channels: - - main - - 9.0 - - 8.0 - -################################################ -# Scheduled Private jobs -################################################ - -# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -- ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: - - # SDK scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - win-x86 - #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available - isPublic: false - jobParameters: - kind: sdk_scenarios - projectFile: sdk_scenarios.proj - channels: - - main - - 9.0 - - 8.0 - - # Blazor 3.2 scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - isPublic: false - jobParameters: - kind: blazor_scenarios - projectFile: blazor_scenarios.proj - channels: - - main - - 9.0 - - 8.0 - - # F# benchmarks - - ${{ if false }}: # skipping, no useful benchmarks there currently - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: fsharp - csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj - runCategories: 'fsharp' - channels: - - main - - 9.0 - - 8.0 - - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: fsharpmicro - csproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj - runCategories: 'FSharpMicro' - channels: - - main - - 9.0 - - 8.0 - - # bepuphysics benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: bepuphysics - csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj - runCategories: 'BepuPhysics' - channels: - - main - - 9.0 - - 8.0 - - # ImageSharp benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: imagesharp - csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj - runCategories: 'ImageSharp' - channels: - - main - - 9.0 - - 8.0 - - # Akade.IndexedSet benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: akadeindexedset - csproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj - runCategories: 'AkadeIndexedSet' - channels: - - main - - 9.0 - - # ML.NET benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: mlnet - csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj - runCategories: 'mldotnet' - channels: - - main - - 9.0 - - 8.0 - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - - # Roslyn benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: roslyn - csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - runCategories: 'roslyn' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - 9.0 - - 8.0 - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - - # ILLink benchmarks - # disabled because of: https://github.com/dotnet/performance/issues/3569 - # - template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/benchmark_jobs.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) - # isPublic: false - # jobParameters: - # kind: illink - # csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj - # runCategories: 'illink' - # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - # - main - # - 8.0 - - # Powershell benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64 - isPublic: false - jobParameters: - kind: powershell - csproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj - runCategories: 'Public Internal' - channels: - - main - - 9.0 - - 8.0 - - # Secret Sync - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 - inputs: - version: 6.x - - - task: DeleteFiles@1 - inputs: - Contents: global.json - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} - -################################################ -# Manually Triggered Job -################################################ - - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 - inputs: - version: 6.x - - - task: DeleteFiles@1 - inputs: - Contents: global.json - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} - -################################################ -# Scheduled Private jobs (Weekly) -################################################ - -# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -- ${{ if and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), contains(variables['Build.QueuedBy'], 'Weekly')) }}: - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 - inputs: - version: 6.x - - - task: DeleteFiles@1 - inputs: - Contents: global.json - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + - template: /eng/pipelines/sdk-perf-jobs.yml + parameters: + ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: + runPublicJobs: true + ${{ if ne(variables['System.TeamProject'], 'public') }}: + ${{ if or(notin(variables['Build.Reason'], 'PullRequest', 'Manual'), parameters.runPrivateJobs) }}: + runPrivateJobs: true + ${{ if or(eq(variables['Build.CronSchedule.DisplayName'], 'Every 12 hours build'), parameters.runScheduledPrivateJobs) }}: + runScheduledPrivateJobs: true + jobParameters: + ${{ if parameters.onlySanityCheck }}: + onlySanityCheck: true + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule', 'Manual')) }}: + # Secret Sync + - job: Synchronize + pool: + name: NetCore1ESPool-Internal-NoMSI + demands: ImageOverride -equals 1es-windows-2019 + steps: + - task: UseDotNet@2 + displayName: Install .NET 6.0 + inputs: + version: 6.x + + - task: DeleteFiles@1 + inputs: + Contents: global.json + + - script: dotnet tool restore + + - task: AzureCLI@2 + inputs: + azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} diff --git a/eng/performance/benchmark_jobs.yml b/eng/performance/benchmark_jobs.yml deleted file mode 100644 index b635404bd65..00000000000 --- a/eng/performance/benchmark_jobs.yml +++ /dev/null @@ -1,65 +0,0 @@ -parameters: - osName: '' # required -- windows | linux | macos - osVersion: '' # required -- OS version - kind: '' # required -- benchmark kind. - architecture: '' # required -- Architecture. Allowed values: x64, x86, arm, arm64 - pool: '' # required -- name of the Helix pool - queue: '' # required -- name of the Helix queue - container: '' # optional -- id of the container - csproj: '' # required -- relative path to csproj with benchmarks - channels: [] # required -- list of channels to download .NET from - runCategories: '' # required -- string of space separated categories supplied to benchmark dotnet - machinePool: '' # required -- Name of perf machine pool (Tiger, Owl, etc) - affinity: '0' # optional -- Affinity bitmask to a specific machine in the pool (e.g. 1, 2, etc) - runEnvVars: [] # optional -- Environment variables to set for the benchmark run in the form of a multi-line object like: "runEnvVars:\n - DOTNET_GCgen0size=1E00000 \n - DOTNET_GCHeapCount=4 \n - DOTNET_GCTotalPhysicalMemory=200000000" - -jobs: -- template: ../common/templates/jobs/jobs.yml - parameters: - enableTelemetry: false - enablePublishBuildArtifacts: true - helixRepo: dotnet/performance - jobs: - - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.machinePool }}' - displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.machinePool }}' - timeoutInMinutes: 320 - variables: - - name: runEnvVarsParam - ${{ if ne(length(parameters.runEnvVars), 0)}}: - value: "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - ${{ if eq(length(parameters.runEnvVars), 0)}}: - value: '' - - name: Python - ${{ if eq(parameters.osName, 'windows') }}: - value: 'py -3' - ${{ if ne(parameters.osName, 'windows') }}: - value: 'python3' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: internalParam - value: --internal - # for private runs, we want to upload perflab data - - group: DotNet-HelixApi-Access - # perflab upload tokens still exist in this variable group - - group: dotnet-benchview - - ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}: - - name: internalParam - value: '' - workspace: - clean: all - pool: ${{ parameters.pool }} - container: ${{ parameters.container }} - strategy: - matrix: - ${{ each channel in parameters.channels }}: - ${{ channel }}: - _Channel: ${{ channel }} - steps: - - checkout: self - clean: true - - script: $(Python) scripts/run_performance_job.py --performance-repo-ci --queue ${{ parameters.queue }} --channel $(_Channel) --architecture ${{ parameters.architecture }} --run-kind ${{ parameters.kind }} --affinity ${{ parameters.affinity }} $(runEnvVarsParam) --os-group ${{ parameters.osName }} $(internalParam) --target-csproj "${{ parameters.csproj }}" --run-categories "${{ parameters.runCategories }}" - displayName: Run run_performance_job.py - env: - HelixAccessToken: '$(HelixApiAccessToken)' - PerfCommandUploadToken: '$(PerfCommandUploadToken)' - PerfCommandUploadTokenLinux: '$(PerfCommandUploadTokenLinux)' - - template: /eng/performance/send-to-helix.yml \ No newline at end of file diff --git a/eng/performance/gc_jobs.yml b/eng/performance/gc_jobs.yml deleted file mode 100644 index 5ff1c78c2b4..00000000000 --- a/eng/performance/gc_jobs.yml +++ /dev/null @@ -1,52 +0,0 @@ -parameters: - osName: '' # required -- windows | linux | macos - osVersion: '' # required -- OS version - kind: '' # required -- benchmark kind. As of today, only "micro" and "mlnet" benchmarks are supported, we plan to add "scenarios" soon - architecture: '' # required -- Architecture. Allowed values: x64, x86, arm, arm64 - pool: '' # required -- name of the Helix pool - queue: '' # required -- name of the Helix queue - container: '' # optional -- id of the container - -jobs: -- template: ../common/templates/jobs/jobs.yml - parameters: - enableTelemetry: false - enablePublishBuildArtifacts: true - helixRepo: dotnet/performance - jobs: - - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}' - displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }}' - timeoutInMinutes: 320 - variables: - - name: Python - value: 'py -3' - - group: DotNet-HelixApi-Access - workspace: - clean: all - pool: - name: ${{ parameters.pool }} - container: ${{ parameters.container }} - - steps: - - checkout: self - clean: true - - script: $(Python) scripts/gcinfra_setup.py - displayName: Set up GCPerf, GCPerfSim, and C Tools - - #Your steps here - - # - template: /eng/performance/send-to-helix.yml - # parameters: - # HelixSource: '$(HelixSourcePrefix)/dotnet/performance/$(Build.SourceBranch)' # Need to update this - # HelixType: 'test/performance_$(_BuildConfig)/' #Need to update this - # HelixAccessToken: $(HelixApiAccessToken) - # HelixTargetQueues: ${{ parameters.queue }} - # HelixPreCommands: $(HelixPreCommand) - # Creator: $(Creator) - # Architecture: ${{ parameters.architecture }} - # TargetCsproj: ${{ parameters.csproj }} - # WorkItemTimeout: 4:00 # 4 hours - # Python: $(Python) - # ${{ if eq(parameters.osName, 'windows') }}: - # WorkItemDirectory: '$(Build.SourcesDirectory)\docs' # Need to update this - # CorrelationPayloadDirectory: '$(Build.SourcesDirectory)\notLocked' # Need to update this diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml deleted file mode 100644 index da5d78463f0..00000000000 --- a/eng/performance/scenarios.yml +++ /dev/null @@ -1,103 +0,0 @@ -parameters: - osName: '' # required -- windows | linux | macos - osVersion: '' # required -- OS version - kind: '' # required -- benchmark kind - architecture: '' # required -- Architecture. Allowed values: x64, x86, arm, arm64 - pool: '' # required -- name of the Helix pool - queue: '' # required -- name of the Helix queue - container: '' # optional -- id of the container - channels: [] # optional (must have dotnetVersionsLinks if not used) -- list of channels to download .NET from - dotnetVersionsLinks: [] # optional alternative to channels that uses 'channel: link' values to scrape the link's json for dotnet_version or version - projectFile: '' # required -- project file to build (current choices: scenarios.proj/sdk_scenarios.proj ) - machinePool: '' # required -- Name of perf machine pool (Tiger, Owl, etc) - affinity: '0' # optional -- Affinity bitmask to a specific machine in the pool as a decimal number (e.g. 1, 2, etc) - runEnvVars: [] # optional -- Environment variables to set for the benchmark run in the form of a multi-line object like: "runEnvVars:\n - DOTNET_GCgen0size=1E00000 \n - DOTNET_GCHeapCount=4 \n - DOTNET_GCTotalPhysicalMemory=200000000" - additionalJobIdentifier: '' # optional -- Additional identifier to append to the job name (no spaces) - runtimeFlavor: '' # optional -- Runtime flavor used for scenarios - hybridGlobalization: false # optional -- Globalization used for scenarios - -jobs: -- template: ../common/templates/jobs/jobs.yml - parameters: - enableTelemetry: false - enablePublishBuildArtifacts: true - helixRepo: dotnet/performance - jobs: - - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.runtimeFlavor }}_${{ parameters.machinePool }}${{ parameters.additionalJobIdentifier }}' - displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.runtimeFlavor }} ${{ parameters.machinePool }} ${{ parameters.additionalJobIdentifier }}' - timeoutInMinutes: 320 - variables: - - name: hybridGlobalizationParam - ${{ if eq(parameters.hybridGlobalization, 'true')}}: - value: '--hybrid-globalization' - ${{ if ne(parameters.hybridGlobalization, 'true')}}: - value: '' - - name: runtimeFlavorParam - ${{ if ne(parameters.runtimeFlavor, '')}}: - value: --runtime-flavor ${{ parameters.runtimeFlavor }} - ${{ if eq(parameters.runtimeFlavor, '')}}: - value: '' - - name: runEnvVarsParam - ${{ if ne(length(parameters.runEnvVars), 0)}}: # if no runEnvVars are specified, don't pass the parameter to ci_setup - value: "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - ${{ if eq(length(parameters.runEnvVars), 0)}}: - value: '' - - name: Python - ${{ if eq(parameters.osName, 'windows') }}: - value: 'py -3' - ${{ if ne(parameters.osName, 'windows') }}: - value: 'python3' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: internalParam - value: "--internal" - - ${{ if and(eq(parameters.osName, 'windows'), ne(parameters.architecture, 'x86')) }}: - - name: PdnPathParameter - value: '--pdn-path "$(Build.ArtifactStagingDirectory)\\PDN.zip"' - - ${{ else }}: - - name: PdnPathParameter - value: '' - - group: DotNet-HelixApi-Access - - group: dotnet-benchview - - ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}: - - name: internalParam - value: '' - - name: PdnPathParameter - value: '' - workspace: - clean: all - pool: ${{ parameters.pool }} - container: ${{ parameters.container }} - strategy: - matrix: - ${{ if ne(length(parameters.channels), 0) }}: - ${{ each channel in parameters.channels }}: - ${{ channel }}: - _Channel: ${{ channel }} - _DotnetVersionParam: '' - ${{ if ne(length(parameters.dotnetVersionsLinks), 0) }}: - ${{ each versionPair in parameters.dotnetVersionsLinks }}: - ${{ versionPair.key }}_Link: - _Channel: ${{ versionPair.key }} - _DotnetVersionParam: --dotnet-version-link ${{ versionPair.value }} - steps: - - checkout: self - clean: true - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: AzureCLI@2 - condition: ne(variables.PdnPathParameter, '') - displayName: 'Download PDN' - inputs: - azureSubscription: '.NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa)' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - az storage blob download --auth-mode login --account-name pvscmdupload --container-name assets --name paint.net.5.0.3.portable.${{ parameters.architecture }}.zip --file $(Build.ArtifactStagingDirectory)/PDN.zip - - script: $(Python) scripts/run_performance_job.py --performance-repo-ci --is-scenario --queue ${{ parameters.queue }} --channel $(_Channel) --architecture ${{ parameters.architecture }} --run-kind ${{ parameters.kind }} --affinity ${{ parameters.affinity }} $(runEnvVarsParam) --os-group ${{ parameters.osName }} $(runtimeFlavorParam) $(hybridGlobalizationParam) $(PdnPathParameter) --os-version ${{ parameters.osVersion }} $(_DotnetVersionParam) $(internalParam) --project-file $(Build.SourcesDirectory)/eng/performance/${{ parameters.projectFile }} - displayName: Run run_performance_job.py - env: - HelixAccessToken: '$(HelixApiAccessToken)' - PerfCommandUploadToken: '$(PerfCommandUploadToken)' - PerfCommandUploadTokenLinux: '$(PerfCommandUploadTokenLinux)' - - template: /eng/performance/send-to-helix.yml - parameters: - ProjectFile: ${{ parameters.projectFile }} diff --git a/eng/performance/send-to-helix.yml b/eng/performance/send-to-helix.yml deleted file mode 100644 index 770959e6ee9..00000000000 --- a/eng/performance/send-to-helix.yml +++ /dev/null @@ -1,19 +0,0 @@ -# All other properties are passed through as environment variables -parameters: - DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - ProjectFile: helix.proj # optional -- override default project used to send benchmarks - -steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 -warnaserror 0 $env:BUILD_SOURCESDIRECTORY\eng\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh --warnaserror false $BUILD_SOURCESDIRECTORY/eng/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - env: # not sure why I need to pass these through even though they already exist as environment variables - BuildConfig: $(_BuildConfig) - SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml b/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml new file mode 100644 index 00000000000..472c27429fa --- /dev/null +++ b/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml @@ -0,0 +1,140 @@ +parameters: + hybridGlobalization: True + runtimeRepoAlias: runtime + performanceRepoAlias: self + jobParameters: {} + +jobs: + - template: /eng/pipelines/performance/templates/perf-ios-scenarios-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + parameters: + hybridGlobalization: ${{ parameters.hybridGlobalization }} + + # run mono iOS scenarios HybridGlobalization + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: mono + platforms: + - osx_x64 + jobParameters: + runtimeType: iOSMono + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj + runKind: ios_scenarios + isScenario: true + logicalMachine: 'perfiphone12mini' + iOSLlvmBuild: False + iOSStripSymbols: False + hybridGlobalization: ${{ parameters.hybridGlobalization }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: mono + platforms: + - osx_x64 + jobParameters: + runtimeType: iOSMono + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj + runKind: ios_scenarios + isScenario: true + logicalMachine: 'perfiphone12mini' + iOSLlvmBuild: False + iOSStripSymbols: True + hybridGlobalization: ${{ parameters.hybridGlobalization }} + additionalJobIdentifier: iOSStripSymbols + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: mono + platforms: + - osx_x64 + jobParameters: + runtimeType: iOSMono + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj + runKind: ios_scenarios + isScenario: true + logicalMachine: 'perfiphone12mini' + iOSLlvmBuild: True + iOSStripSymbols: False + hybridGlobalization: ${{ parameters.hybridGlobalization }} + additionalJobIdentifier: iOSLlvmBuild + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: mono + platforms: + - osx_x64 + jobParameters: + runtimeType: iOSMono + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj + runKind: ios_scenarios + isScenario: true + logicalMachine: 'perfiphone12mini' + iOSLlvmBuild: True + iOSStripSymbols: True + hybridGlobalization: ${{ parameters.hybridGlobalization }} + additionalJobIdentifier: iOSLlvmBuild iOSStripSymbols + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run NativeAOT iOS scenarios HybridGlobalization + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - osx_x64 + jobParameters: + runtimeType: iOSNativeAOT + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj + runKind: ios_scenarios + isScenario: true + logicalMachine: 'perfiphone12mini' + iOSStripSymbols: False + hybridGlobalization: ${{ parameters.hybridGlobalization }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - osx_x64 + jobParameters: + runtimeType: iOSNativeAOT + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj + runKind: ios_scenarios + isScenario: true + logicalMachine: 'perfiphone12mini' + iOSStripSymbols: True + hybridGlobalization: ${{ parameters.hybridGlobalization }} + additionalJobIdentifier: iOSStripSymbols + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/pipelines/runtime-perf-jobs.yml b/eng/pipelines/runtime-perf-jobs.yml new file mode 100644 index 00000000000..427d815b04f --- /dev/null +++ b/eng/pipelines/runtime-perf-jobs.yml @@ -0,0 +1,346 @@ +parameters: + - name: perfBranch # Currently only used for building the BDN App + type: string + default: main + - name: jobParameters + type: object + default: {} + - name: runtimeRepoAlias + type: string + default: runtime + - name: performanceRepoAlias + type: string + default: self + - name: crossgen + type: object + default: + enabled: true + configs: + - windows_x64 + - windows_x86 + - name: viperJitOptRepeat + type: object + default: + enabled: true + configs: + - windows_x64 + - linux_x64 + - name: viperMicro + type: object + default: + enabled: true + configs: + - windows_x64 + - linux_x64 + - name: owlMicro + type: object + default: + enabled: true + configs: + - windows_x64 + - linux_x64 + - name: tigerNoR2R + type: object + default: + enabled: true + configs: + - windows_x64 + - linux_x64 + - name: tigerNoPGO + type: object + default: + enabled: true + configs: + - windows_x64 + - linux_x64 + - name: tigerMicro + type: object + default: + enabled: true + configs: + - windows_x64 + - linux_x64 + - windown_x86 + - linux_musl_x64 + - name: tigerMonoAOT + type: object + default: + enabled: true + configs: + - linux_x64 + - name: tigerMonoInterpreter + type: object + default: + enabled: true + configs: + - linux_x64 + - name: tigerMono + type: object + default: + enabled: true + configs: + - linux_x64 + - name: androidMono + type: object + default: + enabled: true + configs: + - android_arm64 + +jobs: + - template: /eng/pipelines/performance/templates/perf-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + parameters: + perfBranch: ${{ parameters.perfBranch }} + + # Build and run iOS Mono and NativeAOT scenarios + - template: /eng/pipelines/runtime-ios-scenarios-perf-jobs.yml + parameters: + hybridGlobalization: True + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + jobParameters: ${{ parameters.jobParameters }} + + # run android scenarios + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: mono + platforms: + - windows_x64 + jobParameters: + runtimeType: AndroidMono + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/android_scenarios.proj + runKind: android_scenarios + isScenario: true + logicalMachine: 'perfpixel4a' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run mono microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: mono + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: mono + runKind: micro_mono + logicalMachine: 'perftiger' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run mono interpreter perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: mono + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: mono + codeGenType: 'Interpreter' + runKind: micro_mono + logicalMachine: 'perftiger' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run mono aot microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + buildConfig: release + runtimeFlavor: aot + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: mono + codeGenType: 'AOT' + runKind: micro_mono + logicalMachine: 'perftiger' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr perftiger microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + - windows_x64 + - windows_x86 + - linux_musl_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perftiger' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr perftiger microbenchmarks no dynamic pgo perf jobs + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perftiger' + pgoRunType: nodynamicpgo + additionalJobIdentifier: nodynamicpgo + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perftiger' + pgoRunType: nodynamicpgo + additionalJobIdentifier: nodynamicpgo + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr perftiger microbenchmarks no R2R perf jobs + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perftiger' + r2rRunType: nor2r + additionalJobIdentifier: nor2r + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perftiger' + r2rRunType: nor2r + additionalJobIdentifier: nor2r + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr perfowl microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + - windows_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfowl' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr perfviper microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + - windows_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfviper' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr perfviper microbenchmarks perf jitoptrepeat jobs + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + - windows_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfviper' + experimentName: 'jitoptrepeat' + additionalJobIdentifier: jitoptrepeat + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr crossgen perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + - windows_x86 + jobParameters: + liveLibrariesBuildConfig: Release + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj + runKind: crossgen_scenarios + isScenario: true + logicalMachine: 'perftiger_crossgen' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/pipelines/runtime-slow-perf-jobs.yml b/eng/pipelines/runtime-slow-perf-jobs.yml new file mode 100644 index 00000000000..c818bbca2ed --- /dev/null +++ b/eng/pipelines/runtime-slow-perf-jobs.yml @@ -0,0 +1,250 @@ +parameters: + runScheduledJobs: false + runPrivateJobs: false + runtimeRepoAlias: runtime + performanceRepoAlias: self + jobParameters: {} + +jobs: + - ${{ if eq(parameters.runScheduledJobs, true) }}: + - template: /eng/pipelines/performance/templates/perf-arm64-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + parameters: + mono: true + + # run arm64 interpreter jobs for mono + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: mono + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: mono + codeGenType: 'Interpreter' + runKind: micro_mono + logicalMachine: 'perfampere' + timeoutInMinutes: 720 + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run arm64 jit jobs for mono + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: mono + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: mono + runKind: micro_mono + logicalMachine: 'perfampere' + timeoutInMinutes: 720 + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - ${{ if eq(parameters.runPrivateJobs, true) }}: + - template: /eng/pipelines/performance/templates/perf-arm64-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + parameters: + coreclr: true + monoAot: true + + # run mono aot microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + buildConfig: release + runtimeFlavor: aot + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: mono + codeGenType: 'AOT' + runKind: micro_mono + logicalMachine: 'perfampere' + timeoutInMinutes: 780 + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr Linux arm64 ampere microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfampere' + timeoutInMinutes: 780 + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + #run coreclr Linux arm64 ampere no dynamic pgo microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfampere' + timeoutInMinutes: 780 + pgoRunType: nodynamicpgo + additionalJobIdentifier: nodynamicpgo + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + #run coreclr Linux arm64 ampere no R2R microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfampere' + timeoutInMinutes: 780 + r2rRunType: nor2r + additionalJobIdentifier: nor2r + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr Windows arm64 microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfsurf' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr Windows arm64 ampere microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfampere' + timeoutInMinutes: 780 + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr Windows arm64 ampere no dynamic pgo microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfampere' + pgoRunType: nodynamicpgo + timeoutInMinutes: 780 + additionalJobIdentifier: nodynamicpgo + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr Windows arm64 ampere no R2R microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfampere' + r2rRunType: nor2r + timeoutInMinutes: 780 + additionalJobIdentifier: nor2r + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Uncomment once we fix https://github.com/dotnet/performance/issues/1950 + - ${{ if false }}: + # run coreclr linux crossgen perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj + runKind: crossgen_scenarios + isScenario: true + logicalMachine: 'perfa64' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr windows crossgen perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj + runKind: crossgen_scenarios + isScenario: true + logicalMachine: 'perfsurf' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/pipelines/runtime-wasm-perf-jobs.yml b/eng/pipelines/runtime-wasm-perf-jobs.yml new file mode 100644 index 00000000000..a4b32607f52 --- /dev/null +++ b/eng/pipelines/runtime-wasm-perf-jobs.yml @@ -0,0 +1,131 @@ +# This contains all the wasm related perf jobs +parameters: + runProfile: 'non-v8' + downloadSpecificBuild: null # { buildId, pipeline, branchName, project } + runtimeRepoAlias: runtime + performanceRepoAlias: self + jobParameters: {} + +jobs: +- ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: + # build mono on wasm - if not using an existing build + - template: /eng/pipelines/performance/templates/perf-wasm-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + +- ${{ if eq(parameters.runProfile, 'non-v8') }}: + #run mono wasm microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + buildConfig: Release + runtimeFlavor: wasm + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: wasm + codeGenType: 'wasm' + runKind: micro + logicalMachine: 'perftiger' + javascriptEngine: 'javascriptcore' + additionalJobIdentifier: javascriptcore + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + #run mono wasm aot microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # note: should we move this file out of coreclr tempelates because it contains mono jobs? + buildconfig: Release + runtimeflavor: wasm + platforms: + - linux_x64 + jobparameters: + livelibrariesbuildconfig: Release + runtimetype: wasm + codegentype: 'aot' + runkind: micro + logicalMachine: 'perftiger' + javascriptengine: 'javascriptcore' + additionalJobIdentifier: javascriptcore + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.runProfile, 'v8') }}: + # run mono wasm interpreter (default) microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: wasm + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: wasm + codeGenType: 'wasm' + runKind: micro + logicalMachine: 'perftiger' + javascriptEngine: 'v8' + # For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm) + #additionalSetupParameters: '--dotnet-versions 8.0.0' + additionalJobIdentifier: v8 + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + #run mono wasm aot microbenchmarks perf job + # Disabled for runtime-wasm-perf on PRs due to https://github.com/dotnet/runtime/issues/95101 + - ${{if not(in(variables['Build.DefinitionName'], 'runtime-wasm-perf')) }}: + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # note: should we move this file out of coreclr tempelates because it contains mono jobs? + buildconfig: release + runtimeflavor: wasm + platforms: + - linux_x64 + jobparameters: + livelibrariesbuildconfig: Release + runtimetype: wasm + codegentype: 'aot' + runkind: micro + logicalMachine: 'perftiger' + javascriptEngine: 'v8' + # For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm) + #additionalSetupParameters: '--dotnet-versions 8.0.0' # passed to ci_setup.py + additionalJobIdentifier: v8 + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run mono wasm blazor perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: wasm + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: wasm + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/blazor_perf.proj + runKind: blazor_scenarios + isScenario: true + # For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm) + #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to run-performance-job.py + logicalMachine: 'perftiger' + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml new file mode 100644 index 00000000000..6fde591456e --- /dev/null +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -0,0 +1,727 @@ +parameters: + - name: runPublicJobs + type: boolean + default: false + - name: runPrivateJobs + type: boolean + default: false + - name: runScheduledPrivateJobs + type: boolean + default: false + - name: jobParameters + type: object + default: {} + +jobs: +###################################################### +# Public correctness jobs +###################################################### + +- ${{ if parameters.runPublicJobs }}: + + # Scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: scenarios + projectFileName: scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # MAUI scenario benchmarks + - ${{ if false }}: + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: maui_scenarios + projectFileName: maui_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Blazor scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: blazor_scenarios + projectFileName: blazor_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # SDK scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - win-x86 + #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available + isPublic: true + jobParameters: + runKind: sdk_scenarios + projectFileName: sdk_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # micro benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - win-x86 + isPublic: true + jobParameters: + runKind: micro + targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - main + - 9.0 + - 8.0 + + # Ubuntux64 Default and NativeAOT micro benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: micro + targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - main + - nativeaot9.0 + - nativeaot8.0 + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # net462 micro benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-rs5-x64 + isPublic: true + jobParameters: + runKind: micro_net462 + targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - net462 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # ML.NET benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: mlnet + targetCsproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + runCategories: 'mldotnet' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # F# benchmarks + - ${{ if false }}: # skipping, no useful benchmarks there currently + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: fsharp + targetCsproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + runCategories: 'fsharp' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: fsharpmicro + targetCsproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj + runCategories: 'FSharpMicro' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # bepuphysics benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: bepuphysics + targetCsproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + runCategories: 'BepuPhysics' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # ImageSharp benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: imagesharp + targetCsproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + runCategories: 'ImageSharp' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Akade.IndexedSet benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: akadeindexedset + targetCsproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj + runCategories: 'AkadeIndexedSet' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Roslyn benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: roslyn + targetCsproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runCategories: 'roslyn' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # ILLink benchmarks + # disabled because of: https://github.com/dotnet/performance/issues/3569 + - ${{ if false }}: + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: illink + targetCsproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + runCategories: 'illink' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # NativeAOT scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: nativeaot_scenarios + projectFileName: nativeaot_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Powershell benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: powershell + targetCsproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj + runCategories: 'Public' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +########################################### +# Private Jobs +########################################### + +- ${{ if parameters.runPrivateJobs }}: + + # Scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: scenarios + projectFileName: scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Affinitized Scenario benchmarks (Initially just PDN) + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - win-arm64 + - win-arm64-ampere + isPublic: false + jobParameters: + runKind: scenarios + projectFileName: scenarios_affinitized.proj + channels: + - main + - 9.0 + - 8.0 + additionalJobIdentifier: 'Affinity_85' + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + dotnetVersionsLinks: + 9.0: ./eng/Version.Details.xml + channels: + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS Mono scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + dotnetVersionsLinks: + 9.0: ./eng/Version.Details.xml + channels: + - 8.0 + runtimeFlavor: mono + additionalJobIdentifier: Mono + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS Native AOT scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + dotnetVersionsLinks: + 9.0: ./eng/Version.Details.xml + channels: + - 8.0 + runtimeFlavor: coreclr + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui scenario benchmarks + - ${{ if false }}: + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: maui_scenarios + projectFileName: maui_scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # NativeAOT scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + isPublic: false + jobParameters: + runKind: nativeaot_scenarios + projectFileName: nativeaot_scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +################################################ +# Scheduled Private jobs +################################################ + +# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job +- ${{ if parameters.runScheduledPrivateJobs }}: + + # SDK scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - win-x86 + #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available + isPublic: false + jobParameters: + runKind: sdk_scenarios + projectFileName: sdk_scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Blazor 3.2 scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + isPublic: false + jobParameters: + runKind: blazor_scenarios + projectFileName: blazor_scenarios.proj + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # F# benchmarks + - ${{ if false }}: # skipping, no useful benchmarks there currently + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: fsharp + targetCsproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + runCategories: 'fsharp' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: fsharpmicro + targetCsproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj + runCategories: 'FSharpMicro' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # bepuphysics benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: bepuphysics + targetCsproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + runCategories: 'BepuPhysics' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # ImageSharp benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: imagesharp + targetCsproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + runCategories: 'ImageSharp' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Akade.IndexedSet benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: akadeindexedset + targetCsproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj + runCategories: 'AkadeIndexedSet' + channels: + - main + - 9.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # ML.NET benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: mlnet + targetCsproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + runCategories: 'mldotnet' + channels: + - main + - 9.0 + - 8.0 + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Roslyn benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: roslyn + targetCsproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runCategories: 'roslyn' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + - 9.0 + - 8.0 + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # ILLink benchmarks + # disabled because of: https://github.com/dotnet/performance/issues/3569 + - ${{ if false }}: + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) + isPublic: false + jobParameters: + runKind: illink + targetCsproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + runCategories: 'illink' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + - 9.0 + - 8.0 + + # Powershell benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64 + isPublic: false + jobParameters: + runKind: powershell + targetCsproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj + runCategories: 'Public Internal' + channels: + - main + - 9.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/performance/build_machine_matrix.yml b/eng/pipelines/templates/build-machine-matrix.yml similarity index 90% rename from eng/performance/build_machine_matrix.yml rename to eng/pipelines/templates/build-machine-matrix.yml index fba42111f9d..637be851ec4 100644 --- a/eng/performance/build_machine_matrix.yml +++ b/eng/pipelines/templates/build-machine-matrix.yml @@ -8,8 +8,8 @@ jobs: - ${{ if containsValue(parameters.buildMachines, 'win-x64') }}: - template: ${{ parameters.jobTemplate }} parameters: - osName: windows - architecture: x64 + osGroup: windows + archType: x64 pool: vmImage: windows-2019 ${{ insert }}: ${{ parameters.jobParameters }} @@ -25,9 +25,9 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'win-rs5-x64'), eq(parameters.isPublic, true)) }}: # RS5 only used in public builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: windows + osGroup: windows osVersion: RS5 - architecture: x64 + archType: x64 pool: vmImage: windows-2019 machinePool: Open @@ -37,8 +37,8 @@ jobs: - ${{ if containsValue(parameters.buildMachines, 'win-x86') }}: - template: ${{ parameters.jobTemplate }} parameters: - osName: windows - architecture: x86 + osGroup: windows + archType: x86 pool: vmImage: windows-2019 ${{ insert }}: ${{ parameters.jobParameters }} @@ -54,9 +54,9 @@ jobs: - ${{ if containsValue(parameters.buildMachines, 'ubuntu-x64') }}: - template: ${{ parameters.jobTemplate }} parameters: - osName: ubuntu + osGroup: ubuntu osVersion: 2204 - architecture: x64 + archType: x64 pool: vmImage: ubuntu-latest container: ubuntu_x64_build_container @@ -71,9 +71,9 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'win-arm64'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: windows + osGroup: windows osVersion: 20H1 - architecture: arm64 + archType: arm64 pool: vmImage: windows-2019 machinePool: Tiger @@ -83,9 +83,9 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'win-arm64-ampere'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: windows + osGroup: windows osVersion: 22H2 - architecture: arm64 + archType: arm64 pool: vmImage: windows-2019 machinePool: Ampere @@ -95,9 +95,9 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'ubuntu-arm64-ampere'), not(eq(parameters.isPublic, true))) }}: # Ubuntu ARM64 only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: ubuntu + osGroup: ubuntu osVersion: 2204 - architecture: arm64 + archType: arm64 pool: vmImage: ubuntu-latest container: ubuntu_x64_build_container @@ -108,8 +108,8 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'win-x64-android-arm64-pixel'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Pixel only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: windows - architecture: x64 + osGroup: windows + archType: x64 osVersion: 22H2 pool: vmImage: 'windows-2022' @@ -120,8 +120,8 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'win-x64-android-arm64-galaxy'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Galaxy only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: windows - architecture: x64 + osGroup: windows + archType: x64 osVersion: 22H2 pool: vmImage: 'windows-2022' @@ -132,8 +132,8 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'osx-x64-ios-arm64'), not(eq(parameters.isPublic, true))) }}: # iPhone ARM64 12mini only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: osx - architecture: x64 + osGroup: osx + archType: x64 osVersion: 14 pool: vmImage: 'macos-14' diff --git a/eng/pipelines/templates/download-artifact-step.yml b/eng/pipelines/templates/download-artifact-step.yml new file mode 100644 index 00000000000..2470db19de7 --- /dev/null +++ b/eng/pipelines/templates/download-artifact-step.yml @@ -0,0 +1,25 @@ +parameters: + unpackFolder: '' + cleanUnpackFolder: true + artifactFileName: '' + artifactName: '' + displayName: '' + +steps: + # Download artifact + - task: DownloadBuildArtifacts@0 + displayName: 'Download ${{ parameters.displayName }}' + inputs: + buildType: current + downloadType: single + downloadPath: '$(Build.SourcesDirectory)/__download__' + artifactName: '${{ parameters.artifactName }}' + checkDownloadedFiles: true + + # Unzip artifact + - task: ExtractFiles@1 + displayName: 'Unzip ${{ parameters.displayName }}' + inputs: + archiveFilePatterns: $(Build.SourcesDirectory)/__download__/${{ parameters.artifactName }}/**/${{ parameters.artifactFileName }} + destinationFolder: ${{ parameters.unpackFolder }} + cleanDestinationFolder: ${{ parameters.cleanUnpackFolder }} diff --git a/eng/pipelines/templates/download-specific-artifact-step.yml b/eng/pipelines/templates/download-specific-artifact-step.yml new file mode 100644 index 00000000000..9ccf241404a --- /dev/null +++ b/eng/pipelines/templates/download-specific-artifact-step.yml @@ -0,0 +1,33 @@ +parameters: + unpackFolder: '' + cleanUnpackFolder: true + artifactFileName: '' + artifactName: '' + displayName: '' + buildId: '' + branchName: '' + pipeline: '' + project: 'public' # 'internal' or 'public' + +steps: + # Download artifact + - task: DownloadBuildArtifacts@0 + displayName: 'Download specific ${{ parameters.displayName }}' + inputs: + buildType: specific + project: ${{ parameters.project }} + pipeline: ${{ parameters.pipeline }} + buildVersionToDownload: specific + branchName: ${{ parameters.branchName }} + buildId: ${{ parameters.buildId }} + downloadType: single + downloadPath: '$(Build.SourcesDirectory)/__download__' + artifactName: '${{ parameters.artifactName }}' + + # Unzip artifact + - task: ExtractFiles@1 + displayName: 'Unzip specific ${{ parameters.displayName }}' + inputs: + archiveFilePatterns: $(Build.SourcesDirectory)/__download__/${{ parameters.artifactName }}/${{ parameters.artifactFileName }} + destinationFolder: ${{ parameters.unpackFolder }} + cleanDestinationFolder: ${{ parameters.cleanUnpackFolder }} \ No newline at end of file diff --git a/eng/pipelines/templates/run-performance-job-script-step.yml b/eng/pipelines/templates/run-performance-job-script-step.yml new file mode 100644 index 00000000000..0684d84b10f --- /dev/null +++ b/eng/pipelines/templates/run-performance-job-script-step.yml @@ -0,0 +1,11 @@ +parameters: + performanceRepoDir: '' + runPerformanceJobArgs: [] + +steps: + - script: $(Python) ${{ parameters.performanceRepoDir }}/scripts/run_performance_job.py ${{ join(' ', parameters.runPerformanceJobArgs) }} + displayName: Run performance job script + env: + PerfCommandUploadToken: '$(PerfCommandUploadToken)' + PerfCommandUploadTokenLinux: '$(PerfCommandUploadTokenLinux)' + HelixAccessToken: '$(HelixApiAccessToken)' \ No newline at end of file diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml new file mode 100644 index 00000000000..b9266adcf21 --- /dev/null +++ b/eng/pipelines/templates/run-performance-job.yml @@ -0,0 +1,237 @@ +parameters: + # Used to configure the job + jobTemplate: '/eng/common/templates/job/job.yml' + jobTemplateParameters: {} # optional -- additional parameters to pass to the job template + dependsOn: '' # optional -- dependencies of the job + steps: [] # optional -- list of steps to run before the job + pool: '' # required -- name of the Helix pool + container: '' # optional -- id of the container + channels: [] # optional -- list of channels to download .NET from + dotnetVersionsLinks: [] # optional -- alternative to channels that uses 'channel: link' values to scrape the link's json for dotnet_version or version + frameworks: [] # optional -- list of frameworks to run against + additionalJobIdentifier: '' # optional -- Additional job identifier to append to the job name + downloadPdn: false # optional -- Whether or not to download PDN + timeoutInMinutes: 320 # optional -- Timeout for the job + enableTelemetry: true # optional -- Whether to enable telemetry + condition: '' # optional -- Condition to run the job + disableComponentGovernance: false # optional -- Whether to disable component governance + continueOnError: false # optional -- Whether to continue on error + buildConfig: '' # optional -- Build configuration + variables: [] # optional -- Additional variables to set for the job + performanceRepoAlias: self # optional -- The alias for the performance repository + runtimeRepoAlias: runtime # optional -- The alias for the runtime repository + isRuntimeJob: false # optional -- Whether the performance job runs against a build of the runtime repository + checkoutFetchDepth: 20 # optional -- Fetch depth for the checkout + + # Passed into run performance job script + runKind: '' # required -- micro, scenarios, blazor_scenarios ... + archType: '' # required -- Architecture of Helix machine + osGroup: '' # required -- OS of Helix machine + osSubGroup: '' # optional -- OS Subgroup of Helix machine + queue: '' # optional -- Helix queue to run on (required if logicalMachine is not set) + logicalMachine: '' # optional -- Type of Helix machine to use (required if queue is not set) + channel: '' # optional -- .NET channel to test against (use channels). Ignored if channels or dotnetVersionLinks is non-empty + affinity: '0' # optional -- Affinity bitmask to a specific machine in the pool (e.g. 1, 2, etc) + targetCsproj: '' # optional -- Path to the csproj file to run benchmarks on + runCategories: '' # optional -- Categories to run benchmarks on + isScenario: false # optional -- Whether the job is a scenario job + runEnvVars: [] # optional -- Environment variables to set for the benchmark + runtimeFlavor: '' # optional -- Runtime flavor used for scenarios + hybridGlobalization: false # optional -- Globalization used for scenarios + osVersion: '' # optional -- OS version to run on + dotnetVersionLink: '' # optional -- Link to file containing .NET Versions (use dotnetVersionsLinks). Ignored if dotnetVersionsLinks or channels is non-empty + projectFileName: 'helix.proj' # optional -- Name of Helix project file in /eng/performance directory + projectFile: '' # optional -- Full path to project file + framework: '' # optional -- Framework to run against (use frameworks). Ignored if frameworks is non-empty. + codeGenType: '' # optional -- CodeGen type (e.g. JIT, AOT) + runtimeType: '' # optional -- Runtime type (e.g. mono, wasm, coreclr) + useLocalCommitTime: false # optional -- Whether to use the local commit time or fetch it from the remote repository + isCompare: false # optional -- If the job is comparing two configurations + onlySanityCheck: false # optional -- Whether to only run a sanity check for pipeline validation, not performance testing + pgoRunType: '' # optional -- PGO run type (e.g. nodynamicpgo) + r2rRunType: '' # optional -- R2R run type (e.g. nor2r) + experimentName: '' # optional -- Name of the experiment + javascriptEngine: '' # optional -- JavaScript engine to use + iOSLlvmBuild: false # optional -- Whether to build iOS with LLVM + iOSStripSymbols: false # optional -- Whether to strip symbols from the iOS build + additionalSetupParameters: '' # optional -- Additional arguments to pass to the script + +jobs: +- template: ${{ parameters.jobTemplate }} + parameters: + enableTelemetry: ${{ parameters.enableTelemetry }} + name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} + displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), ' NULL', '') }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + workspace: + clean: all + pool: ${{ parameters.pool }} + container: ${{ parameters.container }} + condition: ${{ parameters.condition }} + dependsOn: + - ${{ if ne(parameters.dependsOn, '') }}: + - ${{ parameters.dependsOn }} + continueOnError: ${{ parameters.continueOnError }} + artifacts: + publish: + logs: + name: Logs_$(Agent.JobName)_Attempt_$(System.JobAttempt) + + # Component governance does not work on musl machines + ${{ if eq(parameters.osSubGroup, '_musl') }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + + ${{ if or(ne(length(parameters.channels), 0), ne(length(parameters.dotnetVersionsLinks), 0), ne(length(parameters.frameworks), 0)) }}: + strategy: + matrix: + ${{ if ne(length(parameters.channels), 0) }}: + ${{ each channel in parameters.channels }}: + ${{ channel }}: + _Channel: ${{ channel }} + _DotnetVersionParam: '' + ${{ if ne(length(parameters.dotnetVersionsLinks), 0) }}: + ${{ each versionPair in parameters.dotnetVersionsLinks }}: + ${{ versionPair.key }}_Link: + _Channel: ${{ versionPair.key }} + _DotnetVersionParam: --dotnet-version-link ${{ versionPair.value }} + ${{ if ne(length(parameters.frameworks), 0) }}: + ${{ each framework in parameters.frameworks }}: + ${{ framework }}: + _Framework: ${{ framework }} + ${{ each parameter in parameters.jobTemplateParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + variables: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + # for private runs, we want to upload perflab data + - group: DotNet-HelixApi-Access + # perflab upload tokens still exist in this variable group + - group: dotnet-benchview + - ${{ if eq(parameters.downloadPdn, true) }}: + - name: _pdnPath + value: '$(Build.ArtifactStagingDirectory)/PDN.zip' + # Create a list of all the parts of the job identifier. Will be joined with '_' to create the job name and ' ' to create the display name. This handles runs from both the performance and runtime repository. + - name: buildConfigUpper + ${{ if eq(parameters.buildConfig, 'debug') }}: + value: 'Debug' + ${{ if eq(parameters.buildConfig, 'release') }}: + value: 'Release' + ${{ if eq(parameters.buildConfig, 'checked') }}: + value: 'Checked' + - name: _BuildConfig + value: $(buildConfigUpper) + - name: performanceRepoDir + value: $(Pipeline.Workspace)/s/performance + - name: _projectFile + ${{ if ne(parameters.projectFile, '') }}: + value: ${{ parameters.projectFile }} + ${{ else }}: + value: $(performanceRepoDir)/eng/performance/${{ parameters.projectFileName }} + - name: Python + ${{ if eq(parameters.osGroup, 'windows') }}: + value: 'py -3' + ${{ else }}: + value: 'python3' + - ${{ each variable in parameters.variables }}: + - ${{ variable }} + steps: + - ${{ if eq(parameters.isRuntimeJob, true) }}: + - checkout: ${{ parameters.runtimeRepoAlias }} + path: s + clean: true + fetchDepth: ${{ parameters.checkoutFetchDepth }} + - checkout: ${{ parameters.performanceRepoAlias }} + path: s/performance + clean: true + fetchDepth: ${{ parameters.checkoutFetchDepth }} + - ${{ each step in parameters.steps }}: + - ${{ step }} + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.downloadPdn, true)) }}: + - task: AzureCLI@2 + displayName: 'Download PDN' + inputs: + azureSubscription: '.NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa)' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + az storage blob download --auth-mode login --account-name pvscmdupload --container-name assets --name paint.net.5.0.3.portable.${{ parameters.archType }}.zip --file $(_pdnPath) + - template: /eng/pipelines/templates/run-performance-job-script-step.yml + parameters: + performanceRepoDir: $(performanceRepoDir) + runPerformanceJobArgs: + - '--run-kind ${{ parameters.runKind }}' + - '--architecture ${{ parameters.archType }}' + - '--os-group ${{ parameters.osGroup }}' + - ${{ if ne(parameters.osSubGroup, '') }}: + - '--os-sub-group ${{ parameters.osSubGroup }}' + - ${{ if ne(parameters.queue, '') }}: + - '--queue ${{ parameters.queue }}' + - ${{ if ne(parameters.logicalMachine, '') }}: + - '--logical-machine ${{ parameters.logicalMachine }}' + - ${{ if or(ne(length(parameters.channels), 0), ne(length(parameters.dotnetVersionsLinks), 0)) }}: + - '--performance-repo-ci' + - '--channel $(_Channel)' + - $(_DotnetVersionParam) + - ${{ elseif ne(parameters.channel, '') }}: + - '--performance-repo-ci' + - '--channel ${{ parameters.channel }}' + - ${{ if notIn(parameters.affinity, '', '0') }}: + - '--affinity ${{ parameters.affinity }}' + - ${{ if ne(parameters.targetCsproj, '') }}: + - '--target-csproj "${{ parameters.targetCsproj }}"' + - ${{ if ne(parameters.runCategories, '') }}: + - '--run-categories "${{ parameters.runCategories }}"' + - ${{ if eq(parameters.isScenario, true) }}: + - '--is-scenario' + - ${{ if ne(length(parameters.runEnvVars), 0) }}: + - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" + - ${{ if and(eq(parameters.runKind, 'maui_scenarios_ios'), ne(parameters.runtimeFlavor, '')) }}: + - '--runtime-flavor ${{ parameters.runtimeFlavor }}' + - ${{ if eq(parameters.hybridGlobalization, true) }}: + - '--hybrid-globalization' + - ${{ if ne(parameters.osVersion, '') }}: + - '--os-version ${{ parameters.osVersion }}' + - ${{ if ne(parameters.dotnetVersionLink, '') }}: + - '--dotnet-version-link ${{ parameters.dotnetVersionLink }}' + - ${{ if or(ne(parameters.projectFile, ''), ne(parameters.projectFileName, 'helix.proj')) }}: + - '--project-file $(_projectFile)' + - ${{ if ne(length(parameters.frameworks), 0) }}: + - '--framework $(_Framework)' + - ${{ elseif ne(parameters.framework, '') }}: + - '--framework ${{ parameters.framework }}' + - '--performance-repo-dir $(performanceRepoDir)' + - ${{ if eq(parameters.isRuntimeJob, true) }}: + - '--runtime-repo-dir $(Build.SourcesDirectory)' + - ${{ if ne(parameters.codeGenType, '') }}: + - '--codegen-type ${{ parameters.codeGenType }}' + - ${{ if ne(parameters.runtimeType, '') }}: + - '--runtime-type ${{ parameters.runtimeType }}' + - ${{ if eq(parameters.useLocalCommitTime, true) }}: + - '--use-local-commit-time' + - ${{ if eq(parameters.isCompare, true) }}: + - '--compare' + - ${{ if eq(parameters.onlySanityCheck, true) }}: + - '--only-sanity' + - ${{ if ne(parameters.pgoRunType, '') }}: + - '--pgo-run-type ${{ parameters.pgoRunType }}' + - ${{ if ne(parameters.r2rRunType, '') }}: + - '--r2r-run-type ${{ parameters.r2rRunType }}' + - ${{ if ne(parameters.experimentName, '') }}: + - '--experiment-name ${{ parameters.experimentName }}' + - ${{ if ne(parameters.javascriptEngine, '') }}: + - '--javascript-engine ${{ parameters.javascriptEngine }}' + - ${{ if eq(parameters.iOSLlvmBuild, true) }}: + - '--ios-llvm-build' + - ${{ if eq(parameters.iOSStripSymbols, true) }}: + - '--ios-strip-symbols' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - '--internal' + - ${{ if eq(parameters.downloadPdn, true) }}: + - '--pdn-path "$(_pdnPath)"' + - ${{ if ne(parameters.additionalSetupParameters, '') }}: + - '${{ parameters.additionalSetupParameters }}' + - template: /eng/pipelines/templates/send-to-helix-step.yml + parameters: + osGroup: ${{ parameters.osGroup }} + projectFile: $(_projectFile) \ No newline at end of file diff --git a/eng/pipelines/templates/run-scenarios-job.yml b/eng/pipelines/templates/run-scenarios-job.yml new file mode 100644 index 00000000000..2c606ce9418 --- /dev/null +++ b/eng/pipelines/templates/run-scenarios-job.yml @@ -0,0 +1,12 @@ +parameters: + osGroup: '' # required -- windows | linux | macos + archType: '' # required -- Architecture. Allowed values: x64, x86, arm, arm64 + # Please see run-performance-job.yml for additional parameters that can be specified as they will be forwarded through + +jobs: +- template: /eng/pipelines/templates/run-performance-job.yml + parameters: + isScenario: true + downloadPdn: ${{ and(eq(parameters.osGroup, 'windows'), ne(parameters.archType, 'x86')) }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/pipelines/templates/runtime-perf-job.yml b/eng/pipelines/templates/runtime-perf-job.yml new file mode 100644 index 00000000000..e1d0dca3b3b --- /dev/null +++ b/eng/pipelines/templates/runtime-perf-job.yml @@ -0,0 +1,242 @@ +parameters: + steps: [] + variables: [] + framework: net10.0 # Specify the appropriate framework when running release branches (ie net6.0 for release/6.0) + buildConfig: '' + archType: '' + osGroup: '' + osSubgroup: '' + runtimeFlavor: '' + liveLibrariesBuildConfig: '' + runtimeType: 'coreclr' + codeGenType: 'JIT' + iOSLlvmBuild: 'False' + iOSStripSymbols: 'False' + hybridGlobalization: 'False' + isScenario: false + downloadSpecificBuild: null # buildId, pipeline, branchName, project + crossBuild: false + runtimeRepoAlias: runtime + performanceRepoAlias: self + selfIsRuntime: true + # Please see run-performance-job.yml for additional parameters that can be specified as they will be forwarded through + +jobs: +- template: /eng/pipelines/templates/run-performance-job.yml + parameters: + ${{ each parameter in parameters }}: + ${{ if notIn(parameter.key, 'steps', 'variables', 'framework') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + isRuntimeJob: true + useLocalCommitTime: true + frameworks: + - ${{ parameters.framework }} + + # Test job depends on the corresponding build job + ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: + dependsOn: + - ${{ if not(or(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'iOSNativeAOT', 'wasm'), and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')))) }}: + - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, 'coreclr') }} + - ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}: + - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, 'mono') }} + - ${{ if eq(parameters.runtimeType, 'wasm')}}: + - ${{ format('build_{0}{1}_{2}_{3}_{4}_{5}', 'browser', '', 'wasm', 'linux', parameters.buildConfig, parameters.runtimeType) }} + - ${{ if and(eq(parameters.codeGenType, 'AOT'), ne(parameters.runtimeType, 'wasm'))}}: + - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.codeGenType) }} + - ${{ if eq(parameters.runtimeType, 'AndroidMono')}}: + - ${{ 'build_android_arm64_release_AndroidMono' }} + - ${{ 'Build_ios_arm64_release_PerfBDNApp' }} + - ${{ if eq(parameters.runtimeType, 'iOSMono')}}: + - ${{ 'build_ios_arm64_release_iOSMono' }} + - ${{ if eq(parameters.runtimeType, 'iOSNativeAOT')}}: + - ${{ 'build_ios_arm64_release_iOSNativeAOT' }} + + variables: + - name: _crossBuildPropertyArg + value: '' + - ${{ if eq(parameters.crossBuild, true) }}: + - name: _crossBuildPropertyArg + value: '-cross' + - name: librariesDownloadDir + value: $(Build.SourcesDirectory)/artifacts + - name: builtAppDir + value: $(Build.SourcesDirectory) + - name: librariesOverrideArg + value: '' + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - name: librariesOverrideArg + value: '/p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' + + - name: buildConfigUpper + ${{ if eq(parameters.buildConfig, 'debug') }}: + value: 'Debug' + ${{ if eq(parameters.buildConfig, 'release') }}: + value: 'Release' + ${{ if eq(parameters.buildConfig, 'checked') }}: + value: 'Checked' + + - ${{ if eq(parameters.osGroup, 'windows') }}: + - name: archiveExtension + value: '.zip' + - name: scriptExt + value: '.cmd' + - ${{ else }}: + - name: archiveExtension + value: '.tar.gz' + - name: scriptExt + value: '.sh' + + - ${{ each variable in parameters.variables }}: + - ${{ variable }} + + steps: + - ${{ each step in parameters.steps }}: + - ${{ step }} + - ${{ if eq(parameters.osGroup, 'windows') }}: + - template: /eng/pipelines/common/templates/disable-vsupdate-or-failfast.yml@${{ parameters.runtimeRepoAlias }} + - script: | + $(Build.SourcesDirectory)/eng/common/msbuild.sh $(Build.SourcesDirectory)/eng/testing/performance/add_properties_to_pipeline.proj /t:SetVariables + displayName: Add Properties To Pipeline Env + + - ${{ if eq(parameters.runtimeType, 'wasm') }}: + # Download wasm + - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: + - template: /eng/pipelines/templates/download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/BrowserWasm + artifactFileName: BrowserWasm.tar.gz + artifactName: BrowserWasm + displayName: BrowserWasm + - ${{ if ne(parameters.downloadSpecificBuild.buildId, '') }}: + - template: /eng/pipelines/templates/download-specific-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/BrowserWasm + artifactFileName: BrowserWasm.tar.gz + artifactName: BrowserWasm + displayName: BrowserWasm + ${{ insert }}: ${{ parameters.downloadSpecificBuild }} + + - script: >- + mkdir -p $(librariesDownloadDir)/bin/wasm/wasm-data && + mkdir -p $(librariesDownloadDir)/bin/wasm/dotnet && + cp -r $(librariesDownloadDir)/BrowserWasm/staging/dotnet-latest/* $(librariesDownloadDir)/bin/wasm/dotnet && + cp -r $(librariesDownloadDir)/BrowserWasm/staging/built-nugets $(librariesDownloadDir)/bin/wasm && + cp src/mono/browser/test-main.js $(librariesDownloadDir)/bin/wasm/wasm-data/test-main.js && + find $(librariesDownloadDir)/bin/wasm -type d && + find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \; + displayName: "Create wasm directory (Linux)" + - ${{ elseif eq(parameters.codeGenType, 'AOT') }}: + # Download mono AOT + - template: /eng/pipelines/templates/download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/bin + artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_coreclr$(archiveExtension)' + artifactName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_coreclr' + displayName: 'Runtime artifacts' + + - template: /eng/pipelines/templates/download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/LinuxMonoAOT + artifactFileName: LinuxMonoAOT${{ parameters.archType }}.tar.gz + artifactName: LinuxMonoAOT${{ parameters.archType }} + displayName: AOT Mono Artifacts + + - script: "mkdir -p $(librariesDownloadDir)/bin/aot;mkdir -p $(librariesDownloadDir)/bin/aot/pack;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/mono/linux.${{ parameters.archType }}.Release/cross/linux-${{ parameters.archType }}/* $(librariesDownloadDir)/bin/aot;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/microsoft.netcore.app.runtime.linux-${{ parameters.archType }}/Release/* $(librariesDownloadDir)/bin/aot/pack" + displayName: "Create aot directory (Linux)" + - ${{ elseif in(parameters.runtimeType, 'coreclr', 'mono') }}: + - template: /eng/pipelines/templates/download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/bin + artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_${{ parameters.runtimeType }}$(archiveExtension)' + artifactName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_${{ parameters.runtimeType }}' + displayName: 'Runtime artifacts' + - ${{ elseif eq(parameters.runtimeType, 'AndroidMono')}}: + # Download artifacts for Android Testing + - template: /eng/pipelines/templates/download-artifact-step.yml + parameters: + unpackFolder: $(builtAppDir)/androidHelloWorld + cleanUnpackFolder: false + artifactFileName: 'AndroidMonoarm64.tar.gz' + artifactName: 'AndroidMonoarm64' + displayName: 'Mono Android HelloWorld' + - template: /eng/pipelines/templates/download-artifact-step.yml + parameters: + unpackFolder: $(builtAppDir) + cleanUnpackFolder: false + artifactFileName: 'AndroidBDNApk.tar.gz' + artifactName: 'AndroidBDNApk' + displayName: 'Mono Android BDN Apk' + - ${{ elseif or(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.runtimeType, 'iOSNativeAOT')) }}: + # Download iOSMono and Native AOT tests + - template: /eng/pipelines/templates/download-artifact-step.yml + parameters: + unpackFolder: $(builtAppDir)/iosHelloWorld + cleanUnpackFolder: false + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'False')) }}: + artifactName: 'iOSSampleAppNoLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactFileName: 'iOSSampleAppNoLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'True')) }}: + artifactName: 'iOSSampleAppNoLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactFileName: 'iOSSampleAppNoLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'False')) }}: + artifactName: 'iOSSampleAppLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactFileName: 'iOSSampleAppLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'True')) }}: + artifactName: 'iOSSampleAppLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactFileName: 'iOSSampleAppLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'False')) }}: + artifactName: 'iOSSampleAppSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactFileName: 'iOSSampleAppSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'True')) }}: + artifactName: 'iOSSampleAppNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactFileName: 'iOSSampleAppNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + displayName: 'iOS Sample App' + # same artifact as above but don't extract .zip + - task: DownloadBuildArtifacts@0 + displayName: 'Download iOS Sample App' + inputs: + buildType: current + downloadType: single + downloadPath: '$(builtAppDir)/iosHelloWorldZip' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'False')) }}: + artifactName: 'iOSSampleAppNoLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'True')) }}: + artifactName: 'iOSSampleAppNoLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'False')) }}: + artifactName: 'iOSSampleAppLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'True')) }}: + artifactName: 'iOSSampleAppLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'False')) }}: + artifactName: 'iOSSampleAppSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'True')) }}: + artifactName: 'iOSSampleAppNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + checkDownloadedFiles: true + + - ${{ if notIn(parameters.runtimeType, 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT') }}: + - ${{ if ne(parameters.runtimeFlavor, 'Mono') }}: + # Create Core_Root + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} generatelayoutonly $(librariesOverrideArg) $(_crossBuildPropertyArg) + displayName: Create Core_Root + - ${{ elseif ne(parameters.osGroup, 'windows') }}: + - task: CopyFiles@2 + displayName: Create .dotnet-mono folder + inputs: + SourceFolder: $(librariesDownloadDir)/bin/testhost/${{ parameters.framework }}-${{ parameters.osGroup }}-$(buildConfigUpper)-${{ parameters.archType }} + TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono + - task: CopyFiles@2 + displayName: Copy corerun to .dotnet-mono + inputs: + SourceFolder: $(librariesDownloadDir)/bin/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper) + Contents: 'corerun*' + TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono/shared/Microsoft.NETCore.App/$(productVersion) + + - ${{ if and(eq(parameters.isScenario, true), eq(parameters.runtimeType, 'wasm')) }}: + - script: >- + mkdir -p $(librariesDownloadDir)/bin/wasm/data && + cp -r $(librariesDownloadDir)/BrowserWasm/staging/dotnet-latest $(librariesDownloadDir)/bin/wasm && + cp -r $(librariesDownloadDir)/BrowserWasm/staging/built-nugets $(librariesDownloadDir)/bin/wasm && + cp src/mono/browser/test-main.js $(librariesDownloadDir)/bin/wasm/data/test-main.js && + find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \; + displayName: "Create wasm directory (Linux)" + condition: succeeded() diff --git a/eng/pipelines/templates/send-to-helix-step.yml b/eng/pipelines/templates/send-to-helix-step.yml new file mode 100644 index 00000000000..4fb02049069 --- /dev/null +++ b/eng/pipelines/templates/send-to-helix-step.yml @@ -0,0 +1,23 @@ +# All other properties are passed through as environment variables +parameters: + osGroup: '' # required -- OS group of the build agent + projectFile: '' # required -- Helix project file + condition: '' # optional -- condition for step to execute; defaults to succeeded() + displayName: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO + environment: {} # optional -- additional environment variables to pass to the step + shouldContinueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: +- ${{ if eq(parameters.osGroup, 'windows') }}: + - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 --restore -ci -warnaserror 0 ${{ parameters.projectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$(BuildConfig)\SendToHelix.binlog + displayName: ${{ parameters.displayName }} (Windows) + condition: and(succeeded(), ${{ and(ne(parameters.condition, false), ne(parameters.projectFile, '')) }}) + env: ${{ parameters.environment }} + continueOnError: ${{ eq(parameters.shouldContinueOnError, true) }} + +- ${{ if ne(parameters.osGroup, 'windows') }}: + - script: $(Build.SourcesDirectory)/eng/common/msbuild.sh --restore --ci --warnaserror false ${{ parameters.projectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(BuildConfig)/SendToHelix.binlog + displayName: ${{ parameters.displayName }} (Unix) + condition: and(succeeded(), ${{ and(ne(parameters.condition, false), ne(parameters.projectFile, '')) }}) + env: ${{ parameters.environment }} + continueOnError: ${{ eq(parameters.shouldContinueOnError, true) }} \ No newline at end of file diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index 048689012c4..4ad38b30650 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -82,6 +82,7 @@ class RunPerformanceJobArgs: build_source_branch: str = os.environ.get("BUILD_SOURCEBRANCH", "main") build_number: str = os.environ.get("BUILD_BUILDNUMBER", "local") build_definition_name: Optional[str] = os.environ.get("BUILD_DEFINITIONNAME") + build_reason: Optional[str] = os.environ.get("BUILD_REASON") internal: bool = False pgo_run_type: Optional[str] = None physical_promotion_run_type: Optional[str] = None @@ -469,7 +470,10 @@ def run_performance_job(args: RunPerformanceJobArgs): creator = "dotnet-performance" perf_lab_arguments = [] scenario_arguments = [] - helix_source_prefix = "pr" + if args.build_reason == "PullRequest": + helix_source_prefix = "pr" + else: + helix_source_prefix = "ci" category_exclusions: list[str] = [] @@ -703,7 +707,6 @@ def run_performance_job(args: RunPerformanceJobArgs): if args.built_app_dir is None: if args.runtime_repo_dir is not None: args.built_app_dir = args.runtime_repo_dir - if android_mono: if args.built_app_dir is None: @@ -1214,4 +1217,5 @@ def main(argv: List[str]): return 1 if __name__ == "__main__": - main(sys.argv) + return_code = main(sys.argv) + sys.exit(return_code)