From 45f3fe1ff6b6de5a6d49c7e9bfaa5755b6fa35f5 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Fri, 3 Jan 2025 07:38:15 +1000 Subject: [PATCH 01/45] Move runtime yaml into performance repository --- azure-pipelines.yml | 280 +++++++++--------- eng/performance/benchmark_jobs.yml | 65 ---- eng/performance/gc_jobs.yml | 12 +- eng/performance/scenarios.yml | 103 ------- eng/performance/send-to-helix.yml | 19 -- .../build-machine-matrix.yml} | 40 +-- .../templates/download-artifact-step.yml | 25 ++ .../download-specific-artifact-step.yml | 33 +++ .../templates/run-performance-job.yml | 259 ++++++++++++++++ .../templates/run-scenarios-job.yml | 12 + .../templates/runtime-perf-job.yml | 233 +++++++++++++++ .../templates/send-to-helix-step.yml | 23 ++ scripts/run_performance_job.py | 7 +- 13 files changed, 756 insertions(+), 355 deletions(-) delete mode 100644 eng/performance/benchmark_jobs.yml delete mode 100644 eng/performance/scenarios.yml delete mode 100644 eng/performance/send-to-helix.yml rename eng/performance/{build_machine_matrix.yml => templates/build-machine-matrix.yml} (90%) create mode 100644 eng/performance/templates/download-artifact-step.yml create mode 100644 eng/performance/templates/download-specific-artifact-step.yml create mode 100644 eng/performance/templates/run-performance-job.yml create mode 100644 eng/performance/templates/run-scenarios-job.yml create mode 100644 eng/performance/templates/runtime-perf-job.yml create mode 100644 eng/performance/templates/send-to-helix-step.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6a6593bb439..dcb5afeb73c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -60,92 +60,92 @@ jobs: - ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: # Scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/scenarios.yml + jobTemplate: /eng/performance/templates/run-scenarios-job.yml buildMachines: - win-x64 - ubuntu-x64 isPublic: true jobParameters: - kind: scenarios - projectFile: scenarios.proj + runKind: scenarios + projectFileName: scenarios.proj channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - main - 8.0 ## MAUI scenario benchmarks - #- template: /eng/performance/build_machine_matrix.yml + #- template: /eng/performance/templates/build-machine-matrix.yml # parameters: - # jobTemplate: /eng/performance/scenarios.yml + # jobTemplate: /eng/performance/templates/run-scenarios-job.yml # buildMachines: # - win-x64 # - ubuntu-x64 # isPublic: true # jobParameters: - # kind: maui_scenarios - # projectFile: maui_scenarios.proj + # runKind: maui_scenarios + # projectFileName: maui_scenarios.proj # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks # - main # - 8.0 # Blazor scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/scenarios.yml + jobTemplate: /eng/performance/templates/run-scenarios-job.yml buildMachines: - win-x64 - ubuntu-x64 isPublic: true jobParameters: - kind: blazor_scenarios - projectFile: blazor_scenarios.proj + runKind: blazor_scenarios + projectFileName: blazor_scenarios.proj channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - main - 8.0 # SDK scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/scenarios.yml + jobTemplate: /eng/performance/templates/run-scenarios-job.yml buildMachines: - win-x64 - win-x86 #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available isPublic: true jobParameters: - kind: sdk_scenarios - projectFile: sdk_scenarios.proj + runKind: sdk_scenarios + projectFileName: sdk_scenarios.proj channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - main - 8.0 # micro benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - win-x86 isPublic: true jobParameters: - kind: micro - csproj: src\benchmarks\micro\MicroBenchmarks.csproj + runKind: micro + targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj runCategories: 'runtime libraries' channels: - main - 8.0 # Ubuntux64 Default and NativeAOT micro benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - ubuntu-x64 isPublic: true jobParameters: - kind: micro - csproj: src\benchmarks\micro\MicroBenchmarks.csproj + runKind: micro + targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj runCategories: 'runtime libraries' channels: - main @@ -154,30 +154,30 @@ jobs: - 8.0 # net462 micro benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-rs5-x64 isPublic: true jobParameters: - kind: micro_net462 - csproj: src\benchmarks\micro\MicroBenchmarks.csproj + runKind: micro_net462 + targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj runCategories: 'runtime libraries' channels: - net462 # ML.NET benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 isPublic: true jobParameters: - kind: mlnet - csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + runKind: mlnet + targetCsproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj runCategories: 'mldotnet' channels: - main @@ -185,95 +185,95 @@ jobs: # F# benchmarks - ${{ if false }}: # skipping, no useful benchmarks there currently - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 isPublic: true jobParameters: - kind: fsharp - csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + runKind: fsharp + targetCsproj: src\benchmarks\real-world\FSharp\FSharp.fsproj runCategories: 'fsharp' channels: - main - 8.0 - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 isPublic: true jobParameters: - kind: fsharpmicro - csproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj + runKind: fsharpmicro + targetCsproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj runCategories: 'FSharpMicro' channels: - main - 8.0 # bepuphysics benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 isPublic: true jobParameters: - kind: bepuphysics - csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + runKind: bepuphysics + targetCsproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj runCategories: 'BepuPhysics' channels: - main - 8.0 # ImageSharp benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 isPublic: true jobParameters: - kind: imagesharp - csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + runKind: imagesharp + targetCsproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj runCategories: 'ImageSharp' channels: - main - 8.0 # Akade.IndexedSet benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 isPublic: true jobParameters: - kind: akadeindexedset - csproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj + runKind: akadeindexedset + targetCsproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj runCategories: 'AkadeIndexedSet' channels: - main - 8.0 # Roslyn benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 isPublic: true jobParameters: - kind: roslyn - csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runKind: roslyn + targetCsproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj runCategories: 'roslyn' channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - main @@ -281,46 +281,46 @@ jobs: # ILLink benchmarks # disabled because of: https://github.com/dotnet/performance/issues/3569 - # - template: /eng/performance/build_machine_matrix.yml + # - template: /eng/performance/templates/build-machine-matrix.yml # parameters: - # jobTemplate: /eng/performance/benchmark_jobs.yml + # jobTemplate: /eng/performance/templates/run-performance-job.yml # buildMachines: # - win-x64 # - ubuntu-x64 # isPublic: true # jobParameters: - # kind: illink - # csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + # runKind: illink + # targetCsproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj # runCategories: 'illink' # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only # - main # NativeAOT scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/scenarios.yml + jobTemplate: /eng/performance/templates/run-scenarios-job.yml buildMachines: - win-x64 - ubuntu-x64 isPublic: true jobParameters: - kind: nativeaot_scenarios - projectFile: nativeaot_scenarios.proj + runKind: nativeaot_scenarios + projectFileName: nativeaot_scenarios.proj channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - main - 8.0 # Powershell benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 isPublic: true jobParameters: - kind: powershell - csproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj + runKind: powershell + targetCsproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj runCategories: 'Public' channels: - main @@ -333,9 +333,9 @@ jobs: - ${{ if or(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: # Scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/scenarios.yml + jobTemplate: /eng/performance/templates/run-scenarios-job.yml buildMachines: - win-x64 - ubuntu-x64 @@ -343,24 +343,24 @@ jobs: - ubuntu-arm64-ampere isPublic: false jobParameters: - kind: scenarios - projectFile: scenarios.proj + runKind: scenarios + projectFileName: scenarios.proj channels: - main - 8.0 # Affinitized Scenario benchmarks (Initially just PDN) - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/scenarios.yml + jobTemplate: /eng/performance/templates/run-scenarios-job.yml buildMachines: - win-x64 - win-arm64 - win-arm64-ampere isPublic: false jobParameters: - kind: scenarios - projectFile: scenarios_affinitized.proj + runKind: scenarios + projectFileName: scenarios_affinitized.proj channels: - main - 8.0 @@ -372,31 +372,31 @@ jobs: - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB # Maui Android scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/scenarios.yml + jobTemplate: /eng/performance/templates/run-scenarios-job.yml buildMachines: - win-x64-android-arm64-pixel - win-x64-android-arm64-galaxy isPublic: false jobParameters: - kind: maui_scenarios_android - projectFile: maui_scenarios_android.proj + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj dotnetVersionsLinks: 9.0: ./eng/Version.Details.xml channels: - 8.0 # Maui iOS Mono scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/scenarios.yml + jobTemplate: /eng/performance/templates/run-scenarios-job.yml buildMachines: - osx-x64-ios-arm64 isPublic: false jobParameters: - kind: maui_scenarios_ios - projectFile: maui_scenarios_ios.proj + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj dotnetVersionsLinks: 9.0: ./eng/Version.Details.xml channels: @@ -404,15 +404,15 @@ jobs: runtimeFlavor: mono # Maui iOS Native AOT scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/scenarios.yml + jobTemplate: /eng/performance/templates/run-scenarios-job.yml buildMachines: - osx-x64-ios-arm64 isPublic: false jobParameters: - kind: maui_scenarios_ios - projectFile: maui_scenarios_ios.proj + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj dotnetVersionsLinks: 9.0: ./eng/Version.Details.xml channels: @@ -420,9 +420,9 @@ jobs: runtimeFlavor: coreclr ## Maui scenario benchmarks - #- template: /eng/performance/build_machine_matrix.yml + #- template: /eng/performance/templates/build-machine-matrix.yml # parameters: - # jobTemplate: /eng/performance/scenarios.yml + # jobTemplate: /eng/performance/templates/run-scenarios-job.yml # buildMachines: # - win-x64 # - ubuntu-x64 @@ -430,24 +430,24 @@ jobs: # - ubuntu-arm64-ampere # isPublic: false # jobParameters: - # kind: maui_scenarios - # projectFile: maui_scenarios.proj + # runKind: maui_scenarios + # projectFileName: maui_scenarios.proj # channels: # - main # - 8.0 # NativeAOT scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/scenarios.yml + jobTemplate: /eng/performance/templates/run-scenarios-job.yml buildMachines: - win-x64 - ubuntu-x64 - win-arm64 isPublic: false jobParameters: - kind: nativeaot_scenarios - projectFile: nativeaot_scenarios.proj + runKind: nativeaot_scenarios + projectFileName: nativeaot_scenarios.proj channels: - main - 8.0 @@ -460,40 +460,40 @@ jobs: - ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: # SDK scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/scenarios.yml + jobTemplate: /eng/performance/templates/run-scenarios-job.yml buildMachines: - win-x64 - win-x86 #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available isPublic: false jobParameters: - kind: sdk_scenarios - projectFile: sdk_scenarios.proj + runKind: sdk_scenarios + projectFileName: sdk_scenarios.proj channels: - main - 8.0 # Blazor 3.2 scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/scenarios.yml + jobTemplate: /eng/performance/templates/run-scenarios-job.yml buildMachines: - win-x64 isPublic: false jobParameters: - kind: blazor_scenarios - projectFile: blazor_scenarios.proj + runKind: blazor_scenarios + projectFileName: blazor_scenarios.proj channels: - main - 8.0 # F# benchmarks - ${{ if false }}: # skipping, no useful benchmarks there currently - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 @@ -501,16 +501,16 @@ jobs: - ubuntu-arm64-ampere isPublic: false jobParameters: - kind: fsharp - csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + runKind: fsharp + targetCsproj: src\benchmarks\real-world\FSharp\FSharp.fsproj runCategories: 'fsharp' channels: - main - 8.0 - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 @@ -518,17 +518,17 @@ jobs: - ubuntu-arm64-ampere isPublic: false jobParameters: - kind: fsharpmicro - csproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj + runKind: fsharpmicro + targetCsproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj runCategories: 'FSharpMicro' channels: - main - 8.0 # bepuphysics benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 @@ -536,17 +536,17 @@ jobs: - ubuntu-arm64-ampere isPublic: false jobParameters: - kind: bepuphysics - csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + runKind: bepuphysics + targetCsproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj runCategories: 'BepuPhysics' channels: - main - 8.0 # ImageSharp benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 @@ -554,17 +554,17 @@ jobs: - ubuntu-arm64-ampere isPublic: false jobParameters: - kind: imagesharp - csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + runKind: imagesharp + targetCsproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj runCategories: 'ImageSharp' channels: - main - 8.0 # Akade.IndexedSet benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 @@ -572,16 +572,16 @@ jobs: - ubuntu-arm64-ampere isPublic: false jobParameters: - kind: akadeindexedset - csproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj + runKind: akadeindexedset + targetCsproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj runCategories: 'AkadeIndexedSet' channels: - main # ML.NET benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 @@ -590,8 +590,8 @@ jobs: - ubuntu-arm64-ampere isPublic: false jobParameters: - kind: mlnet - csproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + runKind: mlnet + targetCsproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj runCategories: 'mldotnet' channels: - main @@ -603,9 +603,9 @@ jobs: - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB # Roslyn benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 @@ -614,8 +614,8 @@ jobs: - ubuntu-arm64-ampere isPublic: false jobParameters: - kind: roslyn - csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runKind: roslyn + targetCsproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj runCategories: 'roslyn' channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - main @@ -628,26 +628,26 @@ jobs: # ILLink benchmarks # disabled because of: https://github.com/dotnet/performance/issues/3569 - # - template: /eng/performance/build_machine_matrix.yml + # - template: /eng/performance/templates/build-machine-matrix.yml # parameters: - # jobTemplate: /eng/performance/benchmark_jobs.yml + # jobTemplate: /eng/performance/templates/run-performance-job.yml # buildMachines: # - win-x64 # - ubuntu-x64 # # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) # isPublic: false # jobParameters: - # kind: illink - # csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + # runKind: illink + # targetCsproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj # runCategories: 'illink' # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only # - main # - 8.0 # Powershell benchmarks - - template: /eng/performance/build_machine_matrix.yml + - template: /eng/performance/templates/build-machine-matrix.yml parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml + jobTemplate: /eng/performance/templates/run-performance-job.yml buildMachines: - win-x64 - ubuntu-x64 @@ -655,8 +655,8 @@ jobs: - ubuntu-arm64 isPublic: false jobParameters: - kind: powershell - csproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj + runKind: powershell + targetCsproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj runCategories: 'Public Internal' channels: - main diff --git a/eng/performance/benchmark_jobs.yml b/eng/performance/benchmark_jobs.yml deleted file mode 100644 index b635404bd65..00000000000 --- a/eng/performance/benchmark_jobs.yml +++ /dev/null @@ -1,65 +0,0 @@ -parameters: - osName: '' # required -- windows | linux | macos - osVersion: '' # required -- OS version - kind: '' # required -- benchmark kind. - architecture: '' # required -- Architecture. Allowed values: x64, x86, arm, arm64 - pool: '' # required -- name of the Helix pool - queue: '' # required -- name of the Helix queue - container: '' # optional -- id of the container - csproj: '' # required -- relative path to csproj with benchmarks - channels: [] # required -- list of channels to download .NET from - runCategories: '' # required -- string of space separated categories supplied to benchmark dotnet - machinePool: '' # required -- Name of perf machine pool (Tiger, Owl, etc) - affinity: '0' # optional -- Affinity bitmask to a specific machine in the pool (e.g. 1, 2, etc) - runEnvVars: [] # optional -- Environment variables to set for the benchmark run in the form of a multi-line object like: "runEnvVars:\n - DOTNET_GCgen0size=1E00000 \n - DOTNET_GCHeapCount=4 \n - DOTNET_GCTotalPhysicalMemory=200000000" - -jobs: -- template: ../common/templates/jobs/jobs.yml - parameters: - enableTelemetry: false - enablePublishBuildArtifacts: true - helixRepo: dotnet/performance - jobs: - - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.machinePool }}' - displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.machinePool }}' - timeoutInMinutes: 320 - variables: - - name: runEnvVarsParam - ${{ if ne(length(parameters.runEnvVars), 0)}}: - value: "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - ${{ if eq(length(parameters.runEnvVars), 0)}}: - value: '' - - name: Python - ${{ if eq(parameters.osName, 'windows') }}: - value: 'py -3' - ${{ if ne(parameters.osName, 'windows') }}: - value: 'python3' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: internalParam - value: --internal - # for private runs, we want to upload perflab data - - group: DotNet-HelixApi-Access - # perflab upload tokens still exist in this variable group - - group: dotnet-benchview - - ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}: - - name: internalParam - value: '' - workspace: - clean: all - pool: ${{ parameters.pool }} - container: ${{ parameters.container }} - strategy: - matrix: - ${{ each channel in parameters.channels }}: - ${{ channel }}: - _Channel: ${{ channel }} - steps: - - checkout: self - clean: true - - script: $(Python) scripts/run_performance_job.py --performance-repo-ci --queue ${{ parameters.queue }} --channel $(_Channel) --architecture ${{ parameters.architecture }} --run-kind ${{ parameters.kind }} --affinity ${{ parameters.affinity }} $(runEnvVarsParam) --os-group ${{ parameters.osName }} $(internalParam) --target-csproj "${{ parameters.csproj }}" --run-categories "${{ parameters.runCategories }}" - displayName: Run run_performance_job.py - env: - HelixAccessToken: '$(HelixApiAccessToken)' - PerfCommandUploadToken: '$(PerfCommandUploadToken)' - PerfCommandUploadTokenLinux: '$(PerfCommandUploadTokenLinux)' - - template: /eng/performance/send-to-helix.yml \ No newline at end of file diff --git a/eng/performance/gc_jobs.yml b/eng/performance/gc_jobs.yml index 5ff1c78c2b4..7743518a82f 100644 --- a/eng/performance/gc_jobs.yml +++ b/eng/performance/gc_jobs.yml @@ -1,8 +1,8 @@ parameters: - osName: '' # required -- windows | linux | macos + osGroup: '' # required -- windows | linux | macos osVersion: '' # required -- OS version kind: '' # required -- benchmark kind. As of today, only "micro" and "mlnet" benchmarks are supported, we plan to add "scenarios" soon - architecture: '' # required -- Architecture. Allowed values: x64, x86, arm, arm64 + archType: '' # required -- Architecture. Allowed values: x64, x86, arm, arm64 pool: '' # required -- name of the Helix pool queue: '' # required -- name of the Helix queue container: '' # optional -- id of the container @@ -14,8 +14,8 @@ jobs: enablePublishBuildArtifacts: true helixRepo: dotnet/performance jobs: - - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}' - displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }}' + - job: '${{ parameters.osGroup }}_${{ parameters.osVersion }}_${{ parameters.archType }}_${{ parameters.kind }}' + displayName: '${{ parameters.osGroup }} ${{ parameters.osVersion }} ${{ parameters.archType }} ${{ parameters.kind }}' timeoutInMinutes: 320 variables: - name: Python @@ -43,10 +43,10 @@ jobs: # HelixTargetQueues: ${{ parameters.queue }} # HelixPreCommands: $(HelixPreCommand) # Creator: $(Creator) - # Architecture: ${{ parameters.architecture }} + # Architecture: ${{ parameters.archType }} # TargetCsproj: ${{ parameters.csproj }} # WorkItemTimeout: 4:00 # 4 hours # Python: $(Python) - # ${{ if eq(parameters.osName, 'windows') }}: + # ${{ if eq(parameters.osGroup, 'windows') }}: # WorkItemDirectory: '$(Build.SourcesDirectory)\docs' # Need to update this # CorrelationPayloadDirectory: '$(Build.SourcesDirectory)\notLocked' # Need to update this diff --git a/eng/performance/scenarios.yml b/eng/performance/scenarios.yml deleted file mode 100644 index da5d78463f0..00000000000 --- a/eng/performance/scenarios.yml +++ /dev/null @@ -1,103 +0,0 @@ -parameters: - osName: '' # required -- windows | linux | macos - osVersion: '' # required -- OS version - kind: '' # required -- benchmark kind - architecture: '' # required -- Architecture. Allowed values: x64, x86, arm, arm64 - pool: '' # required -- name of the Helix pool - queue: '' # required -- name of the Helix queue - container: '' # optional -- id of the container - channels: [] # optional (must have dotnetVersionsLinks if not used) -- list of channels to download .NET from - dotnetVersionsLinks: [] # optional alternative to channels that uses 'channel: link' values to scrape the link's json for dotnet_version or version - projectFile: '' # required -- project file to build (current choices: scenarios.proj/sdk_scenarios.proj ) - machinePool: '' # required -- Name of perf machine pool (Tiger, Owl, etc) - affinity: '0' # optional -- Affinity bitmask to a specific machine in the pool as a decimal number (e.g. 1, 2, etc) - runEnvVars: [] # optional -- Environment variables to set for the benchmark run in the form of a multi-line object like: "runEnvVars:\n - DOTNET_GCgen0size=1E00000 \n - DOTNET_GCHeapCount=4 \n - DOTNET_GCTotalPhysicalMemory=200000000" - additionalJobIdentifier: '' # optional -- Additional identifier to append to the job name (no spaces) - runtimeFlavor: '' # optional -- Runtime flavor used for scenarios - hybridGlobalization: false # optional -- Globalization used for scenarios - -jobs: -- template: ../common/templates/jobs/jobs.yml - parameters: - enableTelemetry: false - enablePublishBuildArtifacts: true - helixRepo: dotnet/performance - jobs: - - job: '${{ parameters.osName }}_${{ parameters.osVersion }}_${{ parameters.architecture }}_${{ parameters.kind }}_${{ parameters.runtimeFlavor }}_${{ parameters.machinePool }}${{ parameters.additionalJobIdentifier }}' - displayName: '${{ parameters.osName }} ${{ parameters.osVersion }} ${{ parameters.architecture }} ${{ parameters.kind }} ${{ parameters.runtimeFlavor }} ${{ parameters.machinePool }} ${{ parameters.additionalJobIdentifier }}' - timeoutInMinutes: 320 - variables: - - name: hybridGlobalizationParam - ${{ if eq(parameters.hybridGlobalization, 'true')}}: - value: '--hybrid-globalization' - ${{ if ne(parameters.hybridGlobalization, 'true')}}: - value: '' - - name: runtimeFlavorParam - ${{ if ne(parameters.runtimeFlavor, '')}}: - value: --runtime-flavor ${{ parameters.runtimeFlavor }} - ${{ if eq(parameters.runtimeFlavor, '')}}: - value: '' - - name: runEnvVarsParam - ${{ if ne(length(parameters.runEnvVars), 0)}}: # if no runEnvVars are specified, don't pass the parameter to ci_setup - value: "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - ${{ if eq(length(parameters.runEnvVars), 0)}}: - value: '' - - name: Python - ${{ if eq(parameters.osName, 'windows') }}: - value: 'py -3' - ${{ if ne(parameters.osName, 'windows') }}: - value: 'python3' - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: internalParam - value: "--internal" - - ${{ if and(eq(parameters.osName, 'windows'), ne(parameters.architecture, 'x86')) }}: - - name: PdnPathParameter - value: '--pdn-path "$(Build.ArtifactStagingDirectory)\\PDN.zip"' - - ${{ else }}: - - name: PdnPathParameter - value: '' - - group: DotNet-HelixApi-Access - - group: dotnet-benchview - - ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}: - - name: internalParam - value: '' - - name: PdnPathParameter - value: '' - workspace: - clean: all - pool: ${{ parameters.pool }} - container: ${{ parameters.container }} - strategy: - matrix: - ${{ if ne(length(parameters.channels), 0) }}: - ${{ each channel in parameters.channels }}: - ${{ channel }}: - _Channel: ${{ channel }} - _DotnetVersionParam: '' - ${{ if ne(length(parameters.dotnetVersionsLinks), 0) }}: - ${{ each versionPair in parameters.dotnetVersionsLinks }}: - ${{ versionPair.key }}_Link: - _Channel: ${{ versionPair.key }} - _DotnetVersionParam: --dotnet-version-link ${{ versionPair.value }} - steps: - - checkout: self - clean: true - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - task: AzureCLI@2 - condition: ne(variables.PdnPathParameter, '') - displayName: 'Download PDN' - inputs: - azureSubscription: '.NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa)' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - az storage blob download --auth-mode login --account-name pvscmdupload --container-name assets --name paint.net.5.0.3.portable.${{ parameters.architecture }}.zip --file $(Build.ArtifactStagingDirectory)/PDN.zip - - script: $(Python) scripts/run_performance_job.py --performance-repo-ci --is-scenario --queue ${{ parameters.queue }} --channel $(_Channel) --architecture ${{ parameters.architecture }} --run-kind ${{ parameters.kind }} --affinity ${{ parameters.affinity }} $(runEnvVarsParam) --os-group ${{ parameters.osName }} $(runtimeFlavorParam) $(hybridGlobalizationParam) $(PdnPathParameter) --os-version ${{ parameters.osVersion }} $(_DotnetVersionParam) $(internalParam) --project-file $(Build.SourcesDirectory)/eng/performance/${{ parameters.projectFile }} - displayName: Run run_performance_job.py - env: - HelixAccessToken: '$(HelixApiAccessToken)' - PerfCommandUploadToken: '$(PerfCommandUploadToken)' - PerfCommandUploadTokenLinux: '$(PerfCommandUploadTokenLinux)' - - template: /eng/performance/send-to-helix.yml - parameters: - ProjectFile: ${{ parameters.projectFile }} diff --git a/eng/performance/send-to-helix.yml b/eng/performance/send-to-helix.yml deleted file mode 100644 index 770959e6ee9..00000000000 --- a/eng/performance/send-to-helix.yml +++ /dev/null @@ -1,19 +0,0 @@ -# All other properties are passed through as environment variables -parameters: - DisplayNamePrefix: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO - condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() - continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false - ProjectFile: helix.proj # optional -- override default project used to send benchmarks - -steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 -warnaserror 0 $env:BUILD_SOURCESDIRECTORY\eng\performance\${{ parameters.ProjectFile }} /restore /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' - displayName: ${{ parameters.DisplayNamePrefix }} (Windows) - condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh --warnaserror false $BUILD_SOURCESDIRECTORY/eng/performance/${{ parameters.ProjectFile }} /restore /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog - displayName: ${{ parameters.DisplayNamePrefix }} (Unix) - condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) - continueOnError: ${{ parameters.continueOnError }} - env: # not sure why I need to pass these through even though they already exist as environment variables - BuildConfig: $(_BuildConfig) - SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/performance/build_machine_matrix.yml b/eng/performance/templates/build-machine-matrix.yml similarity index 90% rename from eng/performance/build_machine_matrix.yml rename to eng/performance/templates/build-machine-matrix.yml index fba42111f9d..637be851ec4 100644 --- a/eng/performance/build_machine_matrix.yml +++ b/eng/performance/templates/build-machine-matrix.yml @@ -8,8 +8,8 @@ jobs: - ${{ if containsValue(parameters.buildMachines, 'win-x64') }}: - template: ${{ parameters.jobTemplate }} parameters: - osName: windows - architecture: x64 + osGroup: windows + archType: x64 pool: vmImage: windows-2019 ${{ insert }}: ${{ parameters.jobParameters }} @@ -25,9 +25,9 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'win-rs5-x64'), eq(parameters.isPublic, true)) }}: # RS5 only used in public builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: windows + osGroup: windows osVersion: RS5 - architecture: x64 + archType: x64 pool: vmImage: windows-2019 machinePool: Open @@ -37,8 +37,8 @@ jobs: - ${{ if containsValue(parameters.buildMachines, 'win-x86') }}: - template: ${{ parameters.jobTemplate }} parameters: - osName: windows - architecture: x86 + osGroup: windows + archType: x86 pool: vmImage: windows-2019 ${{ insert }}: ${{ parameters.jobParameters }} @@ -54,9 +54,9 @@ jobs: - ${{ if containsValue(parameters.buildMachines, 'ubuntu-x64') }}: - template: ${{ parameters.jobTemplate }} parameters: - osName: ubuntu + osGroup: ubuntu osVersion: 2204 - architecture: x64 + archType: x64 pool: vmImage: ubuntu-latest container: ubuntu_x64_build_container @@ -71,9 +71,9 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'win-arm64'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: windows + osGroup: windows osVersion: 20H1 - architecture: arm64 + archType: arm64 pool: vmImage: windows-2019 machinePool: Tiger @@ -83,9 +83,9 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'win-arm64-ampere'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: windows + osGroup: windows osVersion: 22H2 - architecture: arm64 + archType: arm64 pool: vmImage: windows-2019 machinePool: Ampere @@ -95,9 +95,9 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'ubuntu-arm64-ampere'), not(eq(parameters.isPublic, true))) }}: # Ubuntu ARM64 only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: ubuntu + osGroup: ubuntu osVersion: 2204 - architecture: arm64 + archType: arm64 pool: vmImage: ubuntu-latest container: ubuntu_x64_build_container @@ -108,8 +108,8 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'win-x64-android-arm64-pixel'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Pixel only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: windows - architecture: x64 + osGroup: windows + archType: x64 osVersion: 22H2 pool: vmImage: 'windows-2022' @@ -120,8 +120,8 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'win-x64-android-arm64-galaxy'), not(eq(parameters.isPublic, true))) }}: # Windows ARM64 Galaxy only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: windows - architecture: x64 + osGroup: windows + archType: x64 osVersion: 22H2 pool: vmImage: 'windows-2022' @@ -132,8 +132,8 @@ jobs: - ${{ if and(containsValue(parameters.buildMachines, 'osx-x64-ios-arm64'), not(eq(parameters.isPublic, true))) }}: # iPhone ARM64 12mini only used in private builds currently - template: ${{ parameters.jobTemplate }} parameters: - osName: osx - architecture: x64 + osGroup: osx + archType: x64 osVersion: 14 pool: vmImage: 'macos-14' diff --git a/eng/performance/templates/download-artifact-step.yml b/eng/performance/templates/download-artifact-step.yml new file mode 100644 index 00000000000..2470db19de7 --- /dev/null +++ b/eng/performance/templates/download-artifact-step.yml @@ -0,0 +1,25 @@ +parameters: + unpackFolder: '' + cleanUnpackFolder: true + artifactFileName: '' + artifactName: '' + displayName: '' + +steps: + # Download artifact + - task: DownloadBuildArtifacts@0 + displayName: 'Download ${{ parameters.displayName }}' + inputs: + buildType: current + downloadType: single + downloadPath: '$(Build.SourcesDirectory)/__download__' + artifactName: '${{ parameters.artifactName }}' + checkDownloadedFiles: true + + # Unzip artifact + - task: ExtractFiles@1 + displayName: 'Unzip ${{ parameters.displayName }}' + inputs: + archiveFilePatterns: $(Build.SourcesDirectory)/__download__/${{ parameters.artifactName }}/**/${{ parameters.artifactFileName }} + destinationFolder: ${{ parameters.unpackFolder }} + cleanDestinationFolder: ${{ parameters.cleanUnpackFolder }} diff --git a/eng/performance/templates/download-specific-artifact-step.yml b/eng/performance/templates/download-specific-artifact-step.yml new file mode 100644 index 00000000000..9ccf241404a --- /dev/null +++ b/eng/performance/templates/download-specific-artifact-step.yml @@ -0,0 +1,33 @@ +parameters: + unpackFolder: '' + cleanUnpackFolder: true + artifactFileName: '' + artifactName: '' + displayName: '' + buildId: '' + branchName: '' + pipeline: '' + project: 'public' # 'internal' or 'public' + +steps: + # Download artifact + - task: DownloadBuildArtifacts@0 + displayName: 'Download specific ${{ parameters.displayName }}' + inputs: + buildType: specific + project: ${{ parameters.project }} + pipeline: ${{ parameters.pipeline }} + buildVersionToDownload: specific + branchName: ${{ parameters.branchName }} + buildId: ${{ parameters.buildId }} + downloadType: single + downloadPath: '$(Build.SourcesDirectory)/__download__' + artifactName: '${{ parameters.artifactName }}' + + # Unzip artifact + - task: ExtractFiles@1 + displayName: 'Unzip specific ${{ parameters.displayName }}' + inputs: + archiveFilePatterns: $(Build.SourcesDirectory)/__download__/${{ parameters.artifactName }}/${{ parameters.artifactFileName }} + destinationFolder: ${{ parameters.unpackFolder }} + cleanDestinationFolder: ${{ parameters.cleanUnpackFolder }} \ No newline at end of file diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml new file mode 100644 index 00000000000..17c7b8c6f83 --- /dev/null +++ b/eng/performance/templates/run-performance-job.yml @@ -0,0 +1,259 @@ +parameters: + # Used to configure the job + jobTemplate: '/eng/common/templates/job/job.yml' + jobTemplateParameters: {} # optional -- additional parameters to pass to the job template + dependsOn: '' # optional -- dependencies of the job + steps: [] # optional -- list of steps to run before the job + pool: '' # required -- name of the Helix pool + container: '' # optional -- id of the container + channels: [] # optional -- list of channels to download .NET from + dotnetVersionsLinks: [] # optional -- alternative to channels that uses 'channel: link' values to scrape the link's json for dotnet_version or version + frameworks: [] # optional -- list of frameworks to run against + additionalJobIdentifier: '' # optional -- Additional job identifier to append to the job name + downloadPdn: false # optional -- Whether or not to download PDN + timeoutInMinutes: 320 # optional -- Timeout for the job + enableTelemetry: true # optional -- Whether to enable telemetry + condition: '' # optional -- Condition to run the job + disableComponentGovernance: false # optional -- Whether to disable component governance + continueOnError: false # optional -- Whether to continue on error + buildConfig: '' # optional -- Build configuration + variables: {} # optional -- Additional variables to set for the job + + # Passed into run performance job script + runKind: '' # required -- micro, scenarios, blazor_scenarios ... + archType: '' # required -- Architecture of Helix machine + osGroup: '' # required -- OS of Helix machine + osSubGroup: '' # optional -- OS Subgroup of Helix machine + queue: '' # optional -- Helix queue to run on (required if logicalMachine is not set) + logicalMachine: '' # optional -- Type of Helix machine to use (required if queue is not set) + channel: '' # optional -- .NET channel to test against (use channels). Ignored if channels or dotnetVersionLinks is non-empty + affinity: '0' # optional -- Affinity bitmask to a specific machine in the pool (e.g. 1, 2, etc) + targetCsproj: '' # optional -- Path to the csproj file to run benchmarks on + runCategories: '' # optional -- Categories to run benchmarks on + isScenario: false # optional -- Whether the job is a scenario job + runEnvVars: [] # optional -- Environment variables to set for the benchmark + runtimeFlavor: '' # optional -- Runtime flavor used for scenarios + hybridGlobalization: false # optional -- Globalization used for scenarios + osVersion: '' # optional -- OS version to run on + dotnetVersionLink: '' # optional -- Link to file containing .NET Versions (use dotnetVersionsLinks). Ignored if dotnetVersionsLinks or channels is non-empty + projectFileName: 'helix.proj' # optional -- Name of Helix project file in /eng/performance directory + projectFile: '' # optional -- Full path to project file + framework: '' # optional -- Framework to run against (use frameworks). Ignored if frameworks is non-empty. + performanceRepoDir: $(Build.SourcesDirectory) # optional -- Path to the performance repo + runtimeRepoDir: '' # optional -- Path to the runtime repo + codeGenType: '' # optional -- CodeGen type (e.g. JIT, AOT) + runtimeType: '' # optional -- Runtime type (e.g. mono, wasm, coreclr) + useLocalCommitTime: false # optional -- Whether to use the local commit time or fetch it from the remote repository + isCompare: false # optional -- If the job is comparing two configurations + onlySanityCheck: false # optional -- Whether to only run a sanity check for pipeline validation, not performance testing + pgoRunType: '' # optional -- PGO run type (e.g. nodynamicpgo) + r2rRunType: '' # optional -- R2R run type (e.g. nor2r) + experimentName: '' # optional -- Name of the experiment + javascriptEngine: '' # optional -- JavaScript engine to use + iOSLlvmBuild: false # optional -- Whether to build iOS with LLVM + iOSStripSymbols: false # optional -- Whether to strip symbols from the iOS build + additionalSetupParameters: '' # optional -- Additional arguments to pass to the script + +jobs: +- template: ${{ parameters.jobTemplate }} + parameters: + enableTelemetry: ${{ parameters.enableTelemetry }} + enablePublishBuildArtifacts: true + name: ${{ join('_', variables._jobIdentifierParts) }} + displayName: 'Performance ${{ join(' ', variables._jobIdentifierParts) }}' + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + workspace: + clean: all + pool: ${{ parameters.pool }} + container: ${{ parameters.container }} + condition: ${{ parameters.condition }} + dependsOn: + - ${{ if ne(parameters.dependsOn, '') }}: + - ${{ parameters.dependsOn }} + continueOnError: ${{ parameters.continueOnError }} + artifacts: + publish: + logs: + name: Builds_${{ join('_', variables._jobIdentifierParts) }} + + # Component governance does not work on musl machines + ${{ if eq(parameters.osSubGroup, '_musl') }}: + disableComponentGovernance: true + ${{ else }}: + disableComponentGovernance: ${{ parameters.disableComponentGovernance }} + + ${{ if or(ne(length(parameters.channels), 0), ne(length(parameters.dotnetVersionsLinks), 0), ne(length(parameters.frameworks), 0)) }}: + strategy: + matrix: + ${{ if ne(length(parameters.channels), 0) }}: + ${{ each channel in parameters.channels }}: + ${{ channel }}: + _Channel: ${{ channel }} + _DotnetVersionParam: '' + ${{ if ne(length(parameters.dotnetVersionsLinks), 0) }}: + ${{ each versionPair in parameters.dotnetVersionsLinks }}: + ${{ versionPair.key }}_Link: + _Channel: ${{ versionPair.key }} + _DotnetVersionParam: ${{ versionPair.value }} + ${{ if ne(length(parameters.frameworks), 0) }}: + ${{ each framework in parameters.frameworks }}: + ${{ framework }}: + _Framework: ${{ framework }} + ${{ each parameter in parameters.jobTemplateParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + variables: + - name: _internal + value: ${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }} + - ${{ if eq(variables._internal, true)}}: + # for private runs, we want to upload perflab data + - group: DotNet-HelixApi-Access + # perflab upload tokens still exist in this variable group + - group: dotnet-benchview + # Create a list of all the parts of the job identifier. Will be joined with '_' to create the job name and ' ' to create the display name. This handles runs from both the performance and runtime repository. + - name: _jobIdentifierParts + value: + - ${{ parameters.osGroup }} + - ${{ if ne(parameters.osSubGroup, '') }}: + - ${{ parameters.osSubGroup }} + - ${{ if ne(parameters.osVersion, '') }}: + - ${{ parameters.osVersion }} + - ${{ parameters.archType }} + - ${{ if and(eq(parameters.isScenario, true), ne(parameters.runtimeFlavor, '')) }}: + - ${{ parameters.runtimeFlavor }} + - ${{ if ne(parameters.runtimeType, '') }}: + - ${{ parameters.runtimeType }} + - ${{ if ne(parameters.codeGenType, '') }}: + - ${{ parameters.codeGenType }} + - ${{ parameters.runKind }} + - ${{ if ne(parameters.machinePool, '') }}: + - ${{ parameters.machinePool }} + - ${{ if ne(parameters.logicalMachine, '') }}: + - ${{ parameters.logicalMachine }} + - ${{ if ne(parameters.javascriptEngine, '') }}: + - ${{ parameters.javascriptEngine }} + - ${{ if ne(parameters.pgoRunType, '') }}: + - ${{ parameters.pgoRunType }} + - ${{ if ne(parameters.r2rRunType, '') }}: + - ${{ parameters.r2rRunType }} + - ${{ if ne(parameters.experimentName, '') }}: + - ${{ parameters.experimentName }} + - ${{ if eq(parameters.iOSLlvmBuild, true) }}: + - 'iOSLlvmBuild' + - ${{ if eq(parameters.iOSStripSymbols, true) }}: + - 'iOSStripSymbols' + - ${{ if eq(parameters.hybridGlobalization, true) }}: + - 'hybridGlobalization' + - ${{ if ne(parameters.additionalJobIdentifier, '') }}: + - ${{ parameters.additionalJobIdentifier }} + - name: _BuildConfig + value: ${{ parameters.buildConfig }} + - name: _projectFile + ${{ if ne(parameters.projectFile, '') }}: + value: ${{ parameters.projectFile }} + ${{ else }}: + value: ${{ parameters.performanceRepoDir }}/eng/performance/${{ parameters.projectFileName }} + - name: _pdnPath + ${{ if and(eq(variables._internal, true), eq(parameters.downloadPdn, true)) }}: + value: '$(Build.ArtifactStagingDirectory)/PDN.zip' + ${{ else }}: + value: '' + - name: _runPerformanceJobArgList + value: + - ${{ if eq(variables._internal, true) }}: + - '--internal' + - '--run-kind ${{ parameters.runKind }}' + - '--architecture ${{ parameters.archType }}' + - '--os-group ${{ parameters.osGroup }}' + - ${{ if ne(parameters.osSubGroup, '') }}: + - '--os-sub-group ${{ parameters.osSubGroup }}' + - ${{ if ne(parameters.queue, '') }}: + - '--queue ${{ parameters.queue }}' + - ${{ if ne(parameters.logicalMachine, '') }}: + - '--logical-machine ${{ parameters.logicalMachine }}' + - ${{ if or(ne(length(parameters.channels), 0), ne(length(parameters.dotnetVersionsLinks), 0)) }}: + - '--performance-repo-ci' + - '--channel $(_Channel)' + - ${{ else if ne(parameters.channel, '') }}: + - '--performance-repo-ci' + - '--channel ${{ parameters.channel }}' + - ${{ if ne(parameters.affinity, '') }}: + - '--affinity ${{ parameters.affinity }}' + - ${{ if ne(parameters.targetCsproj, '') }}: + - '--target-csproj ${{ parameters.targetCsproj }}' + - ${{ if ne(parameters.runCategories, '') }}: + - '--run-categories ${{ parameters.runCategories }}' + - ${{ if eq(parameters.isScenario, true) }}: + - '--is-scenario' + - ${{ if ne(length(parameters.runEnvVars), 0) }}: + - '--run-env-vars ${{ join(' ', parameters.runEnvVars)}}' + - ${{ if ne(parameters.runtimeFlavor, '') }}: + - '--runtime-flavor ${{ parameters.runtimeFlavor }}' + - ${{ if eq(parameters.hybridGlobalization, true) }}: + - '--hybrid-globalization' + - ${{ if ne(parameters.osVersion, '') }}: + - '--os-version ${{ parameters.osVersion }}' + - ${{ if ne(length(parameters.dotnetVersionsLinks), 0) }}: + - '--dotnet-version-link $(_DotnetVersionParam)' + - ${{ elseif ne(parameters.dotnetVersionLink, '') }}: + - '--dotnet-version-link ${{ parameters.dotnetVersionLink }}' + - ${{ if ne(parameters.projectFile, '') }}: + - '--project-file $(_projectFile)' + - ${{ if ne(length(parameters.frameworks), 0) }}: + - '--framework $(_Framework)' + - ${{ elseif ne(parameters.framework, '') }}: + - '--framework ${{ parameters.framework }}' + - ${{ if ne(parameters.performanceRepoDir, '.') }}: + - '--performance-repo-dir ${{ parameters.performanceRepoDir }}' + - ${{ if ne(parameters.runtimeRepoDir, '') }}: + - '--runtime-repo-dir ${{ parameters.runtimeRepoDir }}' + - ${{ if ne(parameters.codeGenType, '') }}: + - '--codegen-type ${{ parameters.codeGenType }}' + - ${{ if ne(parameters.runtimeType, '') }}: + - '--runtime-type ${{ parameters.runtimeType }}' + - ${{ if eq(parameters.useLocalCommitTime, true) }}: + - '--use-local-commit-time' + - ${{ if eq(parameters.isCompare, true) }}: + - '--compare' + - ${{ if eq(parameters.onlySanityCheck, true) }}: + - '--only-sanity' + - ${{ if ne(parameters.pgoRunType, '') }}: + - '--pgo-run-type ${{ parameters.pgoRunType }}' + - ${{ if ne(parameters.r2rRunType, '') }}: + - '--r2r-run-type ${{ parameters.r2rRunType }}' + - ${{ if ne(parameters.experimentName, '') }}: + - '--experiment-name ${{ parameters.experimentName }}' + - ${{ if ne(parameters.javascriptEngine, '') }}: + - '--javascript-engine ${{ parameters.javascriptEngine }}' + - ${{ if eq(parameters.iOSLlvmBuild, true) }}: + - '--ios-llvm-build' + - ${{ if eq(parameters.iOSStripSymbols, true) }}: + - '--ios-strip-symbols' + - ${{ if ne(variables._pdnPath, '') }}: + - '--pdn-path $(_pdnPath)' + - ${{ if ne(parameters.additionalSetupParameters, '') }}: + - '${{ parameters.additionalSetupParameters }}' + - ${{ each variable in parameters.variables }}: + - ${{insert}}: ${{ variable }} + steps: + - ${{ each step in parameters.steps }}: + - ${{ step }} + - ${{ if eq(parameters.downloadPdn, true) }}: + - task: AzureCLI@2 + condition: ne(variables._pdnPath, '') + displayName: 'Download PDN' + inputs: + azureSubscription: '.NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa)' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + az storage blob download --auth-mode login --account-name pvscmdupload --container-name assets --name paint.net.5.0.3.portable.${{ parameters.archType }}.zip --file $(_pdnPath) + - script: $(Python) ${{ parameters.performanceRepoDir }}/scripts/run_performance_job.py ${{ join(' ', variables._runPerformanceJobArgList) }} + displayName: Run performance job script + env: + PerfCommandUploadToken: '$(PerfCommandUploadToken)' + PerfCommandUploadTokenLinux: '$(PerfCommandUploadTokenLinux)' + HelixAccessToken: '$(HelixApiAccessToken)' + - template: ./send-to-helix-step.yml + parameters: + osGroup: ${{ parameters.osGroup }} + projectFile: $(_projectFile) \ No newline at end of file diff --git a/eng/performance/templates/run-scenarios-job.yml b/eng/performance/templates/run-scenarios-job.yml new file mode 100644 index 00000000000..1e0f7c17a50 --- /dev/null +++ b/eng/performance/templates/run-scenarios-job.yml @@ -0,0 +1,12 @@ +parameters: + osGroup: '' # required -- windows | linux | macos + archType: '' # required -- Architecture. Allowed values: x64, x86, arm, arm64 + # Please see run-performance-job.yml for additional parameters that can be specified as they will be forwarded through + +jobs: +- template: ./run-performance-job.yml + parameters: + isScenario: true + downloadPdn: ${{ if and(eq(parameters.osGroup, 'windows'), ne(parameters.archType, 'x86')) }} + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/performance/templates/runtime-perf-job.yml b/eng/performance/templates/runtime-perf-job.yml new file mode 100644 index 00000000000..4283d6a24ce --- /dev/null +++ b/eng/performance/templates/runtime-perf-job.yml @@ -0,0 +1,233 @@ +parameters: + steps: [] + variables: {} + framework: net10.0 # Specify the appropriate framework when running release branches (ie net6.0 for release/6.0) + buildConfig: '' + archType: '' + osGroup: '' + osSubgroup: '' + runtimeFlavor: '' + liveLibrariesBuildConfig: '' + runtimeType: 'coreclr' + codeGenType: 'JIT' + iOSLlvmBuild: 'False' + iOSStripSymbols: 'False' + hybridGlobalization: 'False' + isScenario: false + downloadSpecificBuild: null # buildId, pipeline, branchName, project + crossBuild: false + runtimeRepoDir: '' + performanceRepoDir: '' + # Please see run-performance-job.yml for additional parameters that can be specified as they will be forwarded through + +jobs: +- template: ./run-performance-job.yml + parameters: + ${{ each parameter in parameters }}: + ${{ if notIn(parameter.key, 'steps', 'variables', 'framework') }}: + ${{ parameter.key }}: ${{ parameter.value }} + + useLocalCommitTime: true + frameworks: + - ${{ parameters.framework }} + + # Test job depends on the corresponding build job + ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: + dependsOn: + - ${{ if not(or(in(parameters.runtimeType, 'AndroidMono', 'iOSMono', 'iOSNativeAOT', 'wasm'), and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')))) }}: + - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, 'coreclr') }} + - ${{ if and(eq(parameters.runtimeType, 'mono'), ne(parameters.codeGenType, 'AOT')) }}: + - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, 'mono') }} + - ${{ if eq(parameters.runtimeType, 'wasm')}}: + - ${{ format('build_{0}{1}_{2}_{3}_{4}_{5}', 'browser', '', 'wasm', 'linux', parameters.buildConfig, parameters.runtimeType) }} + - ${{ if and(eq(parameters.codeGenType, 'AOT'), ne(parameters.runtimeType, 'wasm'))}}: + - ${{ format('build_{0}{1}_{2}_{3}_{4}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.codeGenType) }} + - ${{ if eq(parameters.runtimeType, 'AndroidMono')}}: + - ${{ 'build_android_arm64_release_AndroidMono' }} + - ${{ 'Build_ios_arm64_release_PerfBDNApp' }} + - ${{ if eq(parameters.runtimeType, 'iOSMono')}}: + - ${{ 'build_ios_arm64_release_iOSMono' }} + - ${{ if eq(parameters.runtimeType, 'iOSNativeAOT')}}: + - ${{ 'build_ios_arm64_release_iOSNativeAOT' }} + + variables: + - _crossBuildPropertyArg: '' + - ${{ if eq(parameters.crossBuild, true) }}: + - _crossBuildPropertyArg: '-cross' + + - librariesDownloadDir: '${{ parameters.runtimeRepoDir }}/artifacts' + - builtAppDir: ${{ parameters.runtimeRepoDir }} + - librariesOverrideArg: '' + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - librariesOverrideArg : '/p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' + + - name: buildConfigUpper + ${{ if eq(parameters.buildConfig, 'debug') }}: + value: 'Debug' + ${{ if eq(parameters.buildConfig, 'release') }}: + value: 'Release' + ${{ if eq(parameters.buildConfig, 'checked') }}: + value: 'Checked' + + - ${{ if eq(parameters.osGroup, 'windows') }}: + - name: archiveExtension + value: '.zip' + - name: scriptExt + value: '.cmd' + - ${{ else }}: + - name: archiveExtension + value: '.tar.gz' + - name: scriptExt + value: '.sh' + + - ${{ each variable in parameters.variables }}: + - ${{ variable }} + + steps: + - ${{ each step in parameters.steps }}: + - ${{ step }} + - script: | + ${{ parameters.runtimeRepoDir }}/eng/common/msbuild.sh ${{ parameters.runtimeRepoDir }}/eng/testing/performance/add_properties_to_pipeline.proj /t:SetVariables + displayName: Add Properties To Pipeline Env + + - ${{ if eq(parameters.runtimeType, 'wasm') }}: + # Download wasm + - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: + - template: ./download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/BrowserWasm + artifactFileName: BrowserWasm.tar.gz + artifactName: BrowserWasm + displayName: BrowserWasm + - ${{ if ne(parameters.downloadSpecificBuild.buildId, '') }}: + - template: ./download-specific-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/BrowserWasm + artifactFileName: BrowserWasm.tar.gz + artifactName: BrowserWasm + displayName: BrowserWasm + ${{ insert }}: ${{ parameters.downloadSpecificBuild }} + + - script: >- + mkdir -p $(librariesDownloadDir)/bin/wasm/wasm-data && + mkdir -p $(librariesDownloadDir)/bin/wasm/dotnet && + cp -r $(librariesDownloadDir)/BrowserWasm/staging/dotnet-latest/* $(librariesDownloadDir)/bin/wasm/dotnet && + cp -r $(librariesDownloadDir)/BrowserWasm/staging/built-nugets $(librariesDownloadDir)/bin/wasm && + cp src/mono/browser/test-main.js $(librariesDownloadDir)/bin/wasm/wasm-data/test-main.js && + find $(librariesDownloadDir)/bin/wasm -type d && + find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \; + displayName: "Create wasm directory (Linux)" + - ${{ elseif eq(parameters.codeGenType, 'AOT') }}: + # Download mono AOT + - template: ./download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/bin + artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_coreclr$(archiveExtension)' + artifactName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_coreclr' + displayName: 'Runtime artifacts' + + - template: ./download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/LinuxMonoAOT + artifactFileName: LinuxMonoAOT${{ parameters.archType }}.tar.gz + artifactName: LinuxMonoAOT${{ parameters.archType }} + displayName: AOT Mono Artifacts + + - script: "mkdir -p $(librariesDownloadDir)/bin/aot;mkdir -p $(librariesDownloadDir)/bin/aot/pack;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/mono/linux.${{ parameters.archType }}.Release/cross/linux-${{ parameters.archType }}/* $(librariesDownloadDir)/bin/aot;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/microsoft.netcore.app.runtime.linux-${{ parameters.archType }}/Release/* $(librariesDownloadDir)/bin/aot/pack" + displayName: "Create aot directory (Linux)" + - ${{ elseif in(parameters.runtimeType, 'coreclr', 'mono') }}: + - template: ./download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir)/bin + artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.runtimeType }}$(archiveExtension)' + artifactName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.runtimeType }}' + displayName: 'Runtime artifacts' + - ${{ elseif eq(parameters.runtimeType, 'AndroidMono')}}: + # Download artifacts for Android Testing + - template: ./download-artifact-step.yml + parameters: + unpackFolder: $(builtAppDir)/androidHelloWorld + cleanUnpackFolder: false + artifactFileName: 'AndroidMonoarm64.tar.gz' + artifactName: 'AndroidMonoarm64' + displayName: 'Mono Android HelloWorld' + - template: ./download-artifact-step.yml + parameters: + unpackFolder: $(builtAppDir) + cleanUnpackFolder: false + artifactFileName: 'AndroidBDNApk.tar.gz' + artifactName: 'AndroidBDNApk' + displayName: 'Mono Android BDN Apk' + - ${{ elseif or(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.runtimeType, 'iOSNativeAOT')) }}: + # Download iOSMono and Native AOT tests + - template: ./download-artifact-step.yml + parameters: + unpackFolder: $(builtAppDir)/iosHelloWorld + cleanUnpackFolder: false + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'False')) }}: + artifactName: 'iOSSampleAppNoLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactFileName: 'iOSSampleAppNoLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'True')) }}: + artifactName: 'iOSSampleAppNoLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactFileName: 'iOSSampleAppNoLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'False')) }}: + artifactName: 'iOSSampleAppLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactFileName: 'iOSSampleAppLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'True')) }}: + artifactName: 'iOSSampleAppLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactFileName: 'iOSSampleAppLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'False')) }}: + artifactName: 'iOSSampleAppSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactFileName: 'iOSSampleAppSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'True')) }}: + artifactName: 'iOSSampleAppNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactFileName: 'iOSSampleAppNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + displayName: 'iOS Sample App' + # same artifact as above but don't extract .zip + - task: DownloadBuildArtifacts@0 + displayName: 'Download iOS Sample App' + inputs: + buildType: current + downloadType: single + downloadPath: '$(builtAppDir)/iosHelloWorldZip' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'False')) }}: + artifactName: 'iOSSampleAppNoLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'True')) }}: + artifactName: 'iOSSampleAppNoLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'False')) }}: + artifactName: 'iOSSampleAppLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'True')) }}: + artifactName: 'iOSSampleAppLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'False')) }}: + artifactName: 'iOSSampleAppSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'True')) }}: + artifactName: 'iOSSampleAppNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + checkDownloadedFiles: true + + - ${{ if notIn(parameters.runtimeType, 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT') }}: + - ${{ if ne(parameters.runtimeFlavor, 'Mono') }}: + # Create Core_Root + - script: ${{ parameters.runtimeRepoDir }}/src/tests/build$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} generatelayoutonly $(librariesOverrideArg) $(_crossBuildPropertyArg) + displayName: Create Core_Root + - ${{ elseif ne(parameters.osGroup, 'windows') }}: + - task: CopyFiles@2 + displayName: Create .dotnet-mono folder + inputs: + SourceFolder: $(librariesDownloadDir)/bin/testhost/${{ parameters.framework }}-${{ parameters.osGroup }}-$(buildConfigUpper)-${{ parameters.archType }} + TargetFolder: ${{ parameters.runtimeRepoDir }}/.dotnet-mono + - task: CopyFiles@2 + displayName: Copy corerun to .dotnet-mono + inputs: + SourceFolder: $(librariesDownloadDir)/bin/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper) + Contents: 'corerun*' + TargetFolder: ${{ parameters.runtimeRepoDir }}/.dotnet-mono/shared/Microsoft.NETCore.App/$(productVersion) + + - ${{ if and(eq(parameters.isScenario, true), eq(parameters.runtimeType, 'wasm') }}: + - script: >- + mkdir -p $(librariesDownloadDir)/bin/wasm/data && + cp -r $(librariesDownloadDir)/BrowserWasm/staging/dotnet-latest $(librariesDownloadDir)/bin/wasm && + cp -r $(librariesDownloadDir)/BrowserWasm/staging/built-nugets $(librariesDownloadDir)/bin/wasm && + cp src/mono/browser/test-main.js $(librariesDownloadDir)/bin/wasm/data/test-main.js && + find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \; + displayName: "Create wasm directory (Linux)" + condition: succeeded() diff --git a/eng/performance/templates/send-to-helix-step.yml b/eng/performance/templates/send-to-helix-step.yml new file mode 100644 index 00000000000..4fb02049069 --- /dev/null +++ b/eng/performance/templates/send-to-helix-step.yml @@ -0,0 +1,23 @@ +# All other properties are passed through as environment variables +parameters: + osGroup: '' # required -- OS group of the build agent + projectFile: '' # required -- Helix project file + condition: '' # optional -- condition for step to execute; defaults to succeeded() + displayName: 'Send job to Helix' # optional -- rename the beginning of the displayName of the steps in AzDO + environment: {} # optional -- additional environment variables to pass to the step + shouldContinueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false + +steps: +- ${{ if eq(parameters.osGroup, 'windows') }}: + - powershell: $(Build.SourcesDirectory)\eng\common\msbuild.ps1 --restore -ci -warnaserror 0 ${{ parameters.projectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)\artifacts\log\$(BuildConfig)\SendToHelix.binlog + displayName: ${{ parameters.displayName }} (Windows) + condition: and(succeeded(), ${{ and(ne(parameters.condition, false), ne(parameters.projectFile, '')) }}) + env: ${{ parameters.environment }} + continueOnError: ${{ eq(parameters.shouldContinueOnError, true) }} + +- ${{ if ne(parameters.osGroup, 'windows') }}: + - script: $(Build.SourcesDirectory)/eng/common/msbuild.sh --restore --ci --warnaserror false ${{ parameters.projectFile }} /restore /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(BuildConfig)/SendToHelix.binlog + displayName: ${{ parameters.displayName }} (Unix) + condition: and(succeeded(), ${{ and(ne(parameters.condition, false), ne(parameters.projectFile, '')) }}) + env: ${{ parameters.environment }} + continueOnError: ${{ eq(parameters.shouldContinueOnError, true) }} \ No newline at end of file diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index 9ff3684825f..b20b1e2709d 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -82,6 +82,7 @@ class RunPerformanceJobArgs: build_source_branch: str = os.environ.get("BUILD_SOURCEBRANCH", "main") build_number: str = os.environ.get("BUILD_BUILDNUMBER", "local") build_definition_name: Optional[str] = os.environ.get("BUILD_DEFINITIONNAME") + build_reason: Optional[str] = os.environ.get("BUILD_REASON") internal: bool = False pgo_run_type: Optional[str] = None physical_promotion_run_type: Optional[str] = None @@ -469,7 +470,10 @@ def run_performance_job(args: RunPerformanceJobArgs): creator = "dotnet-performance" perf_lab_arguments = [] scenario_arguments = [] - helix_source_prefix = "pr" + if args.build_reason == "PullRequest": + helix_source_prefix = "pr" + else: + helix_source_prefix = "ci" category_exclusions: list[str] = [] @@ -703,7 +707,6 @@ def run_performance_job(args: RunPerformanceJobArgs): if args.built_app_dir is None: if args.runtime_repo_dir is not None: args.built_app_dir = args.runtime_repo_dir - if android_mono: if args.built_app_dir is None: From 83f7e39db13612bdb822438db240552af98a5201 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Fri, 3 Jan 2025 09:03:51 +1000 Subject: [PATCH 02/45] Fix missing end parenthesis --- eng/performance/templates/runtime-perf-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/templates/runtime-perf-job.yml b/eng/performance/templates/runtime-perf-job.yml index 4283d6a24ce..3c0bc4c0bc3 100644 --- a/eng/performance/templates/runtime-perf-job.yml +++ b/eng/performance/templates/runtime-perf-job.yml @@ -222,7 +222,7 @@ jobs: Contents: 'corerun*' TargetFolder: ${{ parameters.runtimeRepoDir }}/.dotnet-mono/shared/Microsoft.NETCore.App/$(productVersion) - - ${{ if and(eq(parameters.isScenario, true), eq(parameters.runtimeType, 'wasm') }}: + - ${{ if and(eq(parameters.isScenario, true), eq(parameters.runtimeType, 'wasm')) }}: - script: >- mkdir -p $(librariesDownloadDir)/bin/wasm/data && cp -r $(librariesDownloadDir)/BrowserWasm/staging/dotnet-latest $(librariesDownloadDir)/bin/wasm && From 594cd74f53653e730a2b991690decdde40b1c9cc Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Fri, 3 Jan 2025 09:09:08 +1000 Subject: [PATCH 03/45] Fix string quoting issues --- eng/performance/templates/run-performance-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 17c7b8c6f83..ff7c63e2453 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -60,7 +60,7 @@ jobs: enableTelemetry: ${{ parameters.enableTelemetry }} enablePublishBuildArtifacts: true name: ${{ join('_', variables._jobIdentifierParts) }} - displayName: 'Performance ${{ join(' ', variables._jobIdentifierParts) }}' + displayName: Performance ${{ join(' ', variables._jobIdentifierParts) }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all @@ -185,7 +185,7 @@ jobs: - ${{ if eq(parameters.isScenario, true) }}: - '--is-scenario' - ${{ if ne(length(parameters.runEnvVars), 0) }}: - - '--run-env-vars ${{ join(' ', parameters.runEnvVars)}}' + - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - ${{ if ne(parameters.runtimeFlavor, '') }}: - '--runtime-flavor ${{ parameters.runtimeFlavor }}' - ${{ if eq(parameters.hybridGlobalization, true) }}: From 3ed6a8495d1720983e5ffe86f5407714f19fe132 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Fri, 3 Jan 2025 09:10:13 +1000 Subject: [PATCH 04/45] Fix elseif --- eng/performance/templates/run-performance-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index ff7c63e2453..c565fdc5a36 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -173,7 +173,7 @@ jobs: - ${{ if or(ne(length(parameters.channels), 0), ne(length(parameters.dotnetVersionsLinks), 0)) }}: - '--performance-repo-ci' - '--channel $(_Channel)' - - ${{ else if ne(parameters.channel, '') }}: + - ${{ elseif ne(parameters.channel, '') }}: - '--performance-repo-ci' - '--channel ${{ parameters.channel }}' - ${{ if ne(parameters.affinity, '') }}: From cb80bad39c54bc04582b553a183c1d6a0a5ae271 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Fri, 3 Jan 2025 09:15:44 +1000 Subject: [PATCH 05/45] Fix variable propagation --- eng/performance/templates/run-performance-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index c565fdc5a36..3d903cbdc41 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -233,7 +233,7 @@ jobs: - ${{ if ne(parameters.additionalSetupParameters, '') }}: - '${{ parameters.additionalSetupParameters }}' - ${{ each variable in parameters.variables }}: - - ${{insert}}: ${{ variable }} + - ${{ variable }} steps: - ${{ each step in parameters.steps }}: - ${{ step }} From 5daf236224d6817fbd8ff7a45dca7c1d9dd00c62 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Fri, 3 Jan 2025 09:22:04 +1000 Subject: [PATCH 06/45] Fix variables to be an array --- eng/performance/templates/run-performance-job.yml | 4 ++-- eng/performance/templates/runtime-perf-job.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 3d903cbdc41..f3d9beb5a01 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -17,7 +17,7 @@ parameters: disableComponentGovernance: false # optional -- Whether to disable component governance continueOnError: false # optional -- Whether to continue on error buildConfig: '' # optional -- Build configuration - variables: {} # optional -- Additional variables to set for the job + variables: [] # optional -- Additional variables to set for the job # Passed into run performance job script runKind: '' # required -- micro, scenarios, blazor_scenarios ... @@ -104,7 +104,7 @@ jobs: variables: - name: _internal value: ${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }} - - ${{ if eq(variables._internal, true)}}: + - ${{ if eq(variables._internal, true) }}: # for private runs, we want to upload perflab data - group: DotNet-HelixApi-Access # perflab upload tokens still exist in this variable group diff --git a/eng/performance/templates/runtime-perf-job.yml b/eng/performance/templates/runtime-perf-job.yml index 3c0bc4c0bc3..6fe758a9839 100644 --- a/eng/performance/templates/runtime-perf-job.yml +++ b/eng/performance/templates/runtime-perf-job.yml @@ -1,6 +1,6 @@ parameters: steps: [] - variables: {} + variables: [] framework: net10.0 # Specify the appropriate framework when running release branches (ie net6.0 for release/6.0) buildConfig: '' archType: '' From ac24b7b625580ef5f721d4b6586fa13b0c1c553c Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Fri, 3 Jan 2025 09:26:59 +1000 Subject: [PATCH 07/45] Another attempt at fixing variables --- .../templates/runtime-perf-job.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/eng/performance/templates/runtime-perf-job.yml b/eng/performance/templates/runtime-perf-job.yml index 6fe758a9839..0fa5a7ef240 100644 --- a/eng/performance/templates/runtime-perf-job.yml +++ b/eng/performance/templates/runtime-perf-job.yml @@ -51,15 +51,20 @@ jobs: - ${{ 'build_ios_arm64_release_iOSNativeAOT' }} variables: - - _crossBuildPropertyArg: '' + - name: _crossBuildPropertyArg + value: '' - ${{ if eq(parameters.crossBuild, true) }}: - - _crossBuildPropertyArg: '-cross' - - - librariesDownloadDir: '${{ parameters.runtimeRepoDir }}/artifacts' - - builtAppDir: ${{ parameters.runtimeRepoDir }} - - librariesOverrideArg: '' + - name: _crossBuildPropertyArg + value: '-cross' + - name: librariesDownloadDir + value: ${{ parameters.runtimeRepoDir }}/artifacts + - name: builtAppDir + value: ${{ parameters.runtimeRepoDir }} + - name: librariesOverrideArg + value: '' - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: - - librariesOverrideArg : '/p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' + - name: librariesOverrideArg + value: '/p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}' - name: buildConfigUpper ${{ if eq(parameters.buildConfig, 'debug') }}: From 24d4b605c791a1f6f7df70378774add2f70f71e6 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Fri, 3 Jan 2025 17:00:18 +1000 Subject: [PATCH 08/45] Fix invalid variables --- .../run-performance-job-script-step.yml | 11 + .../templates/run-performance-job.yml | 227 +++++++++--------- 2 files changed, 119 insertions(+), 119 deletions(-) create mode 100644 eng/performance/templates/run-performance-job-script-step.yml diff --git a/eng/performance/templates/run-performance-job-script-step.yml b/eng/performance/templates/run-performance-job-script-step.yml new file mode 100644 index 00000000000..0684d84b10f --- /dev/null +++ b/eng/performance/templates/run-performance-job-script-step.yml @@ -0,0 +1,11 @@ +parameters: + performanceRepoDir: '' + runPerformanceJobArgs: [] + +steps: + - script: $(Python) ${{ parameters.performanceRepoDir }}/scripts/run_performance_job.py ${{ join(' ', parameters.runPerformanceJobArgs) }} + displayName: Run performance job script + env: + PerfCommandUploadToken: '$(PerfCommandUploadToken)' + PerfCommandUploadTokenLinux: '$(PerfCommandUploadTokenLinux)' + HelixAccessToken: '$(HelixApiAccessToken)' \ No newline at end of file diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index f3d9beb5a01..7e885602ece 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -59,8 +59,8 @@ jobs: parameters: enableTelemetry: ${{ parameters.enableTelemetry }} enablePublishBuildArtifacts: true - name: ${{ join('_', variables._jobIdentifierParts) }} - displayName: Performance ${{ join(' ', variables._jobIdentifierParts) }} + name: ${{ replace(variables._jobName, ' ', '_') }} + displayName: Performance $(_jobName) timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all @@ -74,7 +74,7 @@ jobs: artifacts: publish: logs: - name: Builds_${{ join('_', variables._jobIdentifierParts) }} + name: Builds_${{ replace(variables._jobName, ' ', '_') }} # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: @@ -110,41 +110,28 @@ jobs: # perflab upload tokens still exist in this variable group - group: dotnet-benchview # Create a list of all the parts of the job identifier. Will be joined with '_' to create the job name and ' ' to create the display name. This handles runs from both the performance and runtime repository. - - name: _jobIdentifierParts - value: - - ${{ parameters.osGroup }} - - ${{ if ne(parameters.osSubGroup, '') }}: - - ${{ parameters.osSubGroup }} - - ${{ if ne(parameters.osVersion, '') }}: - - ${{ parameters.osVersion }} - - ${{ parameters.archType }} - - ${{ if and(eq(parameters.isScenario, true), ne(parameters.runtimeFlavor, '')) }}: - - ${{ parameters.runtimeFlavor }} - - ${{ if ne(parameters.runtimeType, '') }}: - - ${{ parameters.runtimeType }} - - ${{ if ne(parameters.codeGenType, '') }}: - - ${{ parameters.codeGenType }} - - ${{ parameters.runKind }} - - ${{ if ne(parameters.machinePool, '') }}: - - ${{ parameters.machinePool }} - - ${{ if ne(parameters.logicalMachine, '') }}: - - ${{ parameters.logicalMachine }} - - ${{ if ne(parameters.javascriptEngine, '') }}: - - ${{ parameters.javascriptEngine }} - - ${{ if ne(parameters.pgoRunType, '') }}: - - ${{ parameters.pgoRunType }} - - ${{ if ne(parameters.r2rRunType, '') }}: - - ${{ parameters.r2rRunType }} - - ${{ if ne(parameters.experimentName, '') }}: - - ${{ parameters.experimentName }} - - ${{ if eq(parameters.iOSLlvmBuild, true) }}: - - 'iOSLlvmBuild' - - ${{ if eq(parameters.iOSStripSymbols, true) }}: - - 'iOSStripSymbols' - - ${{ if eq(parameters.hybridGlobalization, true) }}: - - 'hybridGlobalization' - - ${{ if ne(parameters.additionalJobIdentifier, '') }}: - - ${{ parameters.additionalJobIdentifier }} + - name: _jobNameRuntimeFlavor + ${{ if and(eq(parameters.isScenario, true), ne(parameters.runtimeFlavor, '')) }}: + value: ${{ parameters.runtimeFlavor }} + ${{ else }}: + value: '' + - name: _jobNameIOSLlvmBuild + ${{ if eq(parameters.iOSLlvmBuild, true) }}: + value: iOSLlvmBuild + ${{ else }}: + value: '' + - name: _jobNameIOSStripSymbols + ${{ if eq(parameters.iOSStripSymbols, true) }}: + value: iOSStripSymbols + ${{ else }}: + value: '' + - name: _jobNameHybridGlobalization + ${{ if eq(parameters.hybridGlobalization, true) }}: + value: hybridGlobalization + ${{ else }}: + value: '' + - name: _jobName + value: ${{ replace('${{ parameters.osGroup }} ${{ parameters.osSubGroup }} ${{ parameters.osVersion }} ${{ parameters.archType }} $(_jobNameRuntimeFlavor) ${{ parameters.runtimeType }} ${{ parameters.codeGenType }} ${{ parameters.runKind }} ${{ parameters.machinePool }} ${{ parameters.logicalMachine }} ${{ parameters.javascriptEngine }} ${{ parameters.pgoRunType }} ${{ parameters.r2rRunType }} ${{ parameters.experimentName }} $(_jobNameIOSLlvmBuild) $(_jobNameIOSStripSymbols) $(_jobNameHybridGlobalization) ${{ parameters.additionalJobIdentifier }}', ' ', ' ') }} - name: _BuildConfig value: ${{ parameters.buildConfig }} - name: _projectFile @@ -157,81 +144,12 @@ jobs: value: '$(Build.ArtifactStagingDirectory)/PDN.zip' ${{ else }}: value: '' - - name: _runPerformanceJobArgList - value: - - ${{ if eq(variables._internal, true) }}: - - '--internal' - - '--run-kind ${{ parameters.runKind }}' - - '--architecture ${{ parameters.archType }}' - - '--os-group ${{ parameters.osGroup }}' - - ${{ if ne(parameters.osSubGroup, '') }}: - - '--os-sub-group ${{ parameters.osSubGroup }}' - - ${{ if ne(parameters.queue, '') }}: - - '--queue ${{ parameters.queue }}' - - ${{ if ne(parameters.logicalMachine, '') }}: - - '--logical-machine ${{ parameters.logicalMachine }}' - - ${{ if or(ne(length(parameters.channels), 0), ne(length(parameters.dotnetVersionsLinks), 0)) }}: - - '--performance-repo-ci' - - '--channel $(_Channel)' - - ${{ elseif ne(parameters.channel, '') }}: - - '--performance-repo-ci' - - '--channel ${{ parameters.channel }}' - - ${{ if ne(parameters.affinity, '') }}: - - '--affinity ${{ parameters.affinity }}' - - ${{ if ne(parameters.targetCsproj, '') }}: - - '--target-csproj ${{ parameters.targetCsproj }}' - - ${{ if ne(parameters.runCategories, '') }}: - - '--run-categories ${{ parameters.runCategories }}' - - ${{ if eq(parameters.isScenario, true) }}: - - '--is-scenario' - - ${{ if ne(length(parameters.runEnvVars), 0) }}: - - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - - ${{ if ne(parameters.runtimeFlavor, '') }}: - - '--runtime-flavor ${{ parameters.runtimeFlavor }}' - - ${{ if eq(parameters.hybridGlobalization, true) }}: - - '--hybrid-globalization' - - ${{ if ne(parameters.osVersion, '') }}: - - '--os-version ${{ parameters.osVersion }}' - - ${{ if ne(length(parameters.dotnetVersionsLinks), 0) }}: - - '--dotnet-version-link $(_DotnetVersionParam)' - - ${{ elseif ne(parameters.dotnetVersionLink, '') }}: - - '--dotnet-version-link ${{ parameters.dotnetVersionLink }}' - - ${{ if ne(parameters.projectFile, '') }}: - - '--project-file $(_projectFile)' - - ${{ if ne(length(parameters.frameworks), 0) }}: - - '--framework $(_Framework)' - - ${{ elseif ne(parameters.framework, '') }}: - - '--framework ${{ parameters.framework }}' - - ${{ if ne(parameters.performanceRepoDir, '.') }}: - - '--performance-repo-dir ${{ parameters.performanceRepoDir }}' - - ${{ if ne(parameters.runtimeRepoDir, '') }}: - - '--runtime-repo-dir ${{ parameters.runtimeRepoDir }}' - - ${{ if ne(parameters.codeGenType, '') }}: - - '--codegen-type ${{ parameters.codeGenType }}' - - ${{ if ne(parameters.runtimeType, '') }}: - - '--runtime-type ${{ parameters.runtimeType }}' - - ${{ if eq(parameters.useLocalCommitTime, true) }}: - - '--use-local-commit-time' - - ${{ if eq(parameters.isCompare, true) }}: - - '--compare' - - ${{ if eq(parameters.onlySanityCheck, true) }}: - - '--only-sanity' - - ${{ if ne(parameters.pgoRunType, '') }}: - - '--pgo-run-type ${{ parameters.pgoRunType }}' - - ${{ if ne(parameters.r2rRunType, '') }}: - - '--r2r-run-type ${{ parameters.r2rRunType }}' - - ${{ if ne(parameters.experimentName, '') }}: - - '--experiment-name ${{ parameters.experimentName }}' - - ${{ if ne(parameters.javascriptEngine, '') }}: - - '--javascript-engine ${{ parameters.javascriptEngine }}' - - ${{ if eq(parameters.iOSLlvmBuild, true) }}: - - '--ios-llvm-build' - - ${{ if eq(parameters.iOSStripSymbols, true) }}: - - '--ios-strip-symbols' - - ${{ if ne(variables._pdnPath, '') }}: - - '--pdn-path $(_pdnPath)' - - ${{ if ne(parameters.additionalSetupParameters, '') }}: - - '${{ parameters.additionalSetupParameters }}' + - ${{ if eq(parameters.osGroup, 'windows') }}: + - name: Python + value: 'py -3' + - ${{ if ne(parameters.osGroup, 'windows') }}: + - name: Python + value: 'python3' - ${{ each variable in parameters.variables }}: - ${{ variable }} steps: @@ -247,12 +165,83 @@ jobs: scriptLocation: 'inlineScript' inlineScript: | az storage blob download --auth-mode login --account-name pvscmdupload --container-name assets --name paint.net.5.0.3.portable.${{ parameters.archType }}.zip --file $(_pdnPath) - - script: $(Python) ${{ parameters.performanceRepoDir }}/scripts/run_performance_job.py ${{ join(' ', variables._runPerformanceJobArgList) }} - displayName: Run performance job script - env: - PerfCommandUploadToken: '$(PerfCommandUploadToken)' - PerfCommandUploadTokenLinux: '$(PerfCommandUploadTokenLinux)' - HelixAccessToken: '$(HelixApiAccessToken)' + - template: ./run-performance-job-script-step.yml + parameters: + performanceRepoDir: ${{ parameters.performanceRepoDir }} + runPerformanceJobArgs: + - ${{ if eq(variables._internal, true) }}: + - '--internal' + - '--run-kind ${{ parameters.runKind }}' + - '--architecture ${{ parameters.archType }}' + - '--os-group ${{ parameters.osGroup }}' + - ${{ if ne(parameters.osSubGroup, '') }}: + - '--os-sub-group ${{ parameters.osSubGroup }}' + - ${{ if ne(parameters.queue, '') }}: + - '--queue ${{ parameters.queue }}' + - ${{ if ne(parameters.logicalMachine, '') }}: + - '--logical-machine ${{ parameters.logicalMachine }}' + - ${{ if or(ne(length(parameters.channels), 0), ne(length(parameters.dotnetVersionsLinks), 0)) }}: + - '--performance-repo-ci' + - '--channel $(_Channel)' + - ${{ elseif ne(parameters.channel, '') }}: + - '--performance-repo-ci' + - '--channel ${{ parameters.channel }}' + - ${{ if ne(parameters.affinity, '') }}: + - '--affinity ${{ parameters.affinity }}' + - ${{ if ne(parameters.targetCsproj, '') }}: + - '--target-csproj ${{ parameters.targetCsproj }}' + - ${{ if ne(parameters.runCategories, '') }}: + - '--run-categories ${{ parameters.runCategories }}' + - ${{ if eq(parameters.isScenario, true) }}: + - '--is-scenario' + - ${{ if ne(length(parameters.runEnvVars), 0) }}: + - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" + - ${{ if ne(parameters.runtimeFlavor, '') }}: + - '--runtime-flavor ${{ parameters.runtimeFlavor }}' + - ${{ if eq(parameters.hybridGlobalization, true) }}: + - '--hybrid-globalization' + - ${{ if ne(parameters.osVersion, '') }}: + - '--os-version ${{ parameters.osVersion }}' + - ${{ if ne(length(parameters.dotnetVersionsLinks), 0) }}: + - '--dotnet-version-link $(_DotnetVersionParam)' + - ${{ elseif ne(parameters.dotnetVersionLink, '') }}: + - '--dotnet-version-link ${{ parameters.dotnetVersionLink }}' + - ${{ if ne(parameters.projectFile, '') }}: + - '--project-file $(_projectFile)' + - ${{ if ne(length(parameters.frameworks), 0) }}: + - '--framework $(_Framework)' + - ${{ elseif ne(parameters.framework, '') }}: + - '--framework ${{ parameters.framework }}' + - ${{ if ne(parameters.performanceRepoDir, '.') }}: + - '--performance-repo-dir ${{ parameters.performanceRepoDir }}' + - ${{ if ne(parameters.runtimeRepoDir, '') }}: + - '--runtime-repo-dir ${{ parameters.runtimeRepoDir }}' + - ${{ if ne(parameters.codeGenType, '') }}: + - '--codegen-type ${{ parameters.codeGenType }}' + - ${{ if ne(parameters.runtimeType, '') }}: + - '--runtime-type ${{ parameters.runtimeType }}' + - ${{ if eq(parameters.useLocalCommitTime, true) }}: + - '--use-local-commit-time' + - ${{ if eq(parameters.isCompare, true) }}: + - '--compare' + - ${{ if eq(parameters.onlySanityCheck, true) }}: + - '--only-sanity' + - ${{ if ne(parameters.pgoRunType, '') }}: + - '--pgo-run-type ${{ parameters.pgoRunType }}' + - ${{ if ne(parameters.r2rRunType, '') }}: + - '--r2r-run-type ${{ parameters.r2rRunType }}' + - ${{ if ne(parameters.experimentName, '') }}: + - '--experiment-name ${{ parameters.experimentName }}' + - ${{ if ne(parameters.javascriptEngine, '') }}: + - '--javascript-engine ${{ parameters.javascriptEngine }}' + - ${{ if eq(parameters.iOSLlvmBuild, true) }}: + - '--ios-llvm-build' + - ${{ if eq(parameters.iOSStripSymbols, true) }}: + - '--ios-strip-symbols' + - ${{ if ne(variables._pdnPath, '') }}: + - '--pdn-path $(_pdnPath)' + - ${{ if ne(parameters.additionalSetupParameters, '') }}: + - '${{ parameters.additionalSetupParameters }}' - template: ./send-to-helix-step.yml parameters: osGroup: ${{ parameters.osGroup }} From 0b3ccc1c82038c889c50c9cfb22d259ac8b83e9b Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Tue, 7 Jan 2025 09:24:25 +1000 Subject: [PATCH 09/45] Fix template paths --- eng/performance/templates/runtime-perf-job.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/performance/templates/runtime-perf-job.yml b/eng/performance/templates/runtime-perf-job.yml index 0fa5a7ef240..52c4b40a7a3 100644 --- a/eng/performance/templates/runtime-perf-job.yml +++ b/eng/performance/templates/runtime-perf-job.yml @@ -21,7 +21,7 @@ parameters: # Please see run-performance-job.yml for additional parameters that can be specified as they will be forwarded through jobs: -- template: ./run-performance-job.yml +- template: /eng/performance/templates/run-performance-job.yml parameters: ${{ each parameter in parameters }}: ${{ if notIn(parameter.key, 'steps', 'variables', 'framework') }}: @@ -98,14 +98,14 @@ jobs: - ${{ if eq(parameters.runtimeType, 'wasm') }}: # Download wasm - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: - - template: ./download-artifact-step.yml + - template: /eng/performance/templates/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/BrowserWasm artifactFileName: BrowserWasm.tar.gz artifactName: BrowserWasm displayName: BrowserWasm - ${{ if ne(parameters.downloadSpecificBuild.buildId, '') }}: - - template: ./download-specific-artifact-step.yml + - template: /eng/performance/templates/download-specific-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/BrowserWasm artifactFileName: BrowserWasm.tar.gz @@ -124,14 +124,14 @@ jobs: displayName: "Create wasm directory (Linux)" - ${{ elseif eq(parameters.codeGenType, 'AOT') }}: # Download mono AOT - - template: ./download-artifact-step.yml + - template: /eng/performance/templates/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/bin artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_coreclr$(archiveExtension)' artifactName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_coreclr' displayName: 'Runtime artifacts' - - template: ./download-artifact-step.yml + - template: /eng/performance/templates/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/LinuxMonoAOT artifactFileName: LinuxMonoAOT${{ parameters.archType }}.tar.gz @@ -141,7 +141,7 @@ jobs: - script: "mkdir -p $(librariesDownloadDir)/bin/aot;mkdir -p $(librariesDownloadDir)/bin/aot/pack;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/mono/linux.${{ parameters.archType }}.Release/cross/linux-${{ parameters.archType }}/* $(librariesDownloadDir)/bin/aot;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/microsoft.netcore.app.runtime.linux-${{ parameters.archType }}/Release/* $(librariesDownloadDir)/bin/aot/pack" displayName: "Create aot directory (Linux)" - ${{ elseif in(parameters.runtimeType, 'coreclr', 'mono') }}: - - template: ./download-artifact-step.yml + - template: /eng/performance/templates/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/bin artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.runtimeType }}$(archiveExtension)' @@ -149,14 +149,14 @@ jobs: displayName: 'Runtime artifacts' - ${{ elseif eq(parameters.runtimeType, 'AndroidMono')}}: # Download artifacts for Android Testing - - template: ./download-artifact-step.yml + - template: /eng/performance/templates/download-artifact-step.yml parameters: unpackFolder: $(builtAppDir)/androidHelloWorld cleanUnpackFolder: false artifactFileName: 'AndroidMonoarm64.tar.gz' artifactName: 'AndroidMonoarm64' displayName: 'Mono Android HelloWorld' - - template: ./download-artifact-step.yml + - template: /eng/performance/templates/download-artifact-step.yml parameters: unpackFolder: $(builtAppDir) cleanUnpackFolder: false @@ -165,7 +165,7 @@ jobs: displayName: 'Mono Android BDN Apk' - ${{ elseif or(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.runtimeType, 'iOSNativeAOT')) }}: # Download iOSMono and Native AOT tests - - template: ./download-artifact-step.yml + - template: /eng/performance/templates/download-artifact-step.yml parameters: unpackFolder: $(builtAppDir)/iosHelloWorld cleanUnpackFolder: false From 57343969ecd9ebdafea9fc0ddbdc336f2944cec0 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Tue, 7 Jan 2025 09:26:21 +1000 Subject: [PATCH 10/45] Fix more template paths --- eng/performance/templates/run-performance-job.yml | 4 ++-- eng/performance/templates/run-scenarios-job.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 7e885602ece..124a43ab5b0 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -165,7 +165,7 @@ jobs: scriptLocation: 'inlineScript' inlineScript: | az storage blob download --auth-mode login --account-name pvscmdupload --container-name assets --name paint.net.5.0.3.portable.${{ parameters.archType }}.zip --file $(_pdnPath) - - template: ./run-performance-job-script-step.yml + - template: /eng/performance/templates/run-performance-job-script-step.yml parameters: performanceRepoDir: ${{ parameters.performanceRepoDir }} runPerformanceJobArgs: @@ -242,7 +242,7 @@ jobs: - '--pdn-path $(_pdnPath)' - ${{ if ne(parameters.additionalSetupParameters, '') }}: - '${{ parameters.additionalSetupParameters }}' - - template: ./send-to-helix-step.yml + - template: /eng/performance/templates/send-to-helix-step.yml parameters: osGroup: ${{ parameters.osGroup }} projectFile: $(_projectFile) \ No newline at end of file diff --git a/eng/performance/templates/run-scenarios-job.yml b/eng/performance/templates/run-scenarios-job.yml index 1e0f7c17a50..f47e936c14d 100644 --- a/eng/performance/templates/run-scenarios-job.yml +++ b/eng/performance/templates/run-scenarios-job.yml @@ -4,7 +4,7 @@ parameters: # Please see run-performance-job.yml for additional parameters that can be specified as they will be forwarded through jobs: -- template: ./run-performance-job.yml +- template: /eng/performance/templates/run-performance-job.yml parameters: isScenario: true downloadPdn: ${{ if and(eq(parameters.osGroup, 'windows'), ne(parameters.archType, 'x86')) }} From 0b0965b7a9c6d5b991d660fe548223aa9d22fc9c Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Tue, 7 Jan 2025 09:32:21 +1000 Subject: [PATCH 11/45] Fix indentation --- eng/performance/templates/runtime-perf-job.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/eng/performance/templates/runtime-perf-job.yml b/eng/performance/templates/runtime-perf-job.yml index 52c4b40a7a3..657397bf3ac 100644 --- a/eng/performance/templates/runtime-perf-job.yml +++ b/eng/performance/templates/runtime-perf-job.yml @@ -228,11 +228,11 @@ jobs: TargetFolder: ${{ parameters.runtimeRepoDir }}/.dotnet-mono/shared/Microsoft.NETCore.App/$(productVersion) - ${{ if and(eq(parameters.isScenario, true), eq(parameters.runtimeType, 'wasm')) }}: - - script: >- - mkdir -p $(librariesDownloadDir)/bin/wasm/data && - cp -r $(librariesDownloadDir)/BrowserWasm/staging/dotnet-latest $(librariesDownloadDir)/bin/wasm && - cp -r $(librariesDownloadDir)/BrowserWasm/staging/built-nugets $(librariesDownloadDir)/bin/wasm && - cp src/mono/browser/test-main.js $(librariesDownloadDir)/bin/wasm/data/test-main.js && - find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \; - displayName: "Create wasm directory (Linux)" - condition: succeeded() + - script: >- + mkdir -p $(librariesDownloadDir)/bin/wasm/data && + cp -r $(librariesDownloadDir)/BrowserWasm/staging/dotnet-latest $(librariesDownloadDir)/bin/wasm && + cp -r $(librariesDownloadDir)/BrowserWasm/staging/built-nugets $(librariesDownloadDir)/bin/wasm && + cp src/mono/browser/test-main.js $(librariesDownloadDir)/bin/wasm/data/test-main.js && + find $(librariesDownloadDir)/bin/wasm -type f -exec chmod 664 {} \; + displayName: "Create wasm directory (Linux)" + condition: succeeded() From b8d8cf15fca37e754e07e78f41681588719e2bfa Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Tue, 7 Jan 2025 10:27:33 +1000 Subject: [PATCH 12/45] Try fix the job name --- eng/performance/templates/run-performance-job.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 124a43ab5b0..c0691b02caf 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -60,7 +60,7 @@ jobs: enableTelemetry: ${{ parameters.enableTelemetry }} enablePublishBuildArtifacts: true name: ${{ replace(variables._jobName, ' ', '_') }} - displayName: Performance $(_jobName) + displayName: Performance ${{ variables._jobName }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all @@ -130,8 +130,10 @@ jobs: value: hybridGlobalization ${{ else }}: value: '' + - name: _jobNameRaw + value: ${{ parameters.osGroup }} ${{ parameters.osSubGroup }} ${{ parameters.osVersion }} ${{ parameters.archType }} ${{ variables._jobNameRuntimeFlavor }} ${{ parameters.runtimeType }} ${{ parameters.codeGenType }} ${{ parameters.runKind }} ${{ parameters.machinePool }} ${{ parameters.logicalMachine }} ${{ parameters.javascriptEngine }} ${{ parameters.pgoRunType }} ${{ parameters.r2rRunType }} ${{ parameters.experimentName }} ${{ variables._jobNameIOSLlvmBuild }} ${{ variables._jobNameIOSStripSymbols }} ${{ variables._jobNameHybridGlobalization }} ${{ parameters.additionalJobIdentifier }} - name: _jobName - value: ${{ replace('${{ parameters.osGroup }} ${{ parameters.osSubGroup }} ${{ parameters.osVersion }} ${{ parameters.archType }} $(_jobNameRuntimeFlavor) ${{ parameters.runtimeType }} ${{ parameters.codeGenType }} ${{ parameters.runKind }} ${{ parameters.machinePool }} ${{ parameters.logicalMachine }} ${{ parameters.javascriptEngine }} ${{ parameters.pgoRunType }} ${{ parameters.r2rRunType }} ${{ parameters.experimentName }} $(_jobNameIOSLlvmBuild) $(_jobNameIOSStripSymbols) $(_jobNameHybridGlobalization) ${{ parameters.additionalJobIdentifier }}', ' ', ' ') }} + value: ${{ replace(variables._jobNameRaw, ' ', ' ') }} - name: _BuildConfig value: ${{ parameters.buildConfig }} - name: _projectFile From f3a610f9b61afa2f7fd6d8a318cd8cc9330b2cc2 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Tue, 7 Jan 2025 13:00:29 +1000 Subject: [PATCH 13/45] Use variables correctly --- .../templates/run-performance-job.yml | 73 +++++++++---------- .../templates/run-scenarios-job.yml | 2 +- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index c0691b02caf..73b196de96a 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -54,6 +54,32 @@ parameters: iOSStripSymbols: false # optional -- Whether to strip symbols from the iOS build additionalSetupParameters: '' # optional -- Additional arguments to pass to the script +variables: + - name: _jobNameRuntimeFlavor + ${{ if and(eq(parameters.isScenario, true), ne(parameters.runtimeFlavor, '')) }}: + value: ${{ parameters.runtimeFlavor }} + ${{ else }}: + value: '' + - name: _jobNameIOSLlvmBuild + ${{ if eq(parameters.iOSLlvmBuild, true) }}: + value: iOSLlvmBuild + ${{ else }}: + value: '' + - name: _jobNameIOSStripSymbols + ${{ if eq(parameters.iOSStripSymbols, true) }}: + value: iOSStripSymbols + ${{ else }}: + value: '' + - name: _jobNameHybridGlobalization + ${{ if eq(parameters.hybridGlobalization, true) }}: + value: hybridGlobalization + ${{ else }}: + value: '' + - name: _jobNameRaw + value: ${{ parameters.osGroup }} ${{ parameters.osSubGroup }} ${{ parameters.osVersion }} ${{ parameters.archType }} ${{ variables._jobNameRuntimeFlavor }} ${{ parameters.runtimeType }} ${{ parameters.codeGenType }} ${{ parameters.runKind }} ${{ parameters.machinePool }} ${{ parameters.logicalMachine }} ${{ parameters.javascriptEngine }} ${{ parameters.pgoRunType }} ${{ parameters.r2rRunType }} ${{ parameters.experimentName }} ${{ variables._jobNameIOSLlvmBuild }} ${{ variables._jobNameIOSStripSymbols }} ${{ variables._jobNameHybridGlobalization }} ${{ parameters.additionalJobIdentifier }} + - name: _jobName + value: ${{ replace(variables._jobNameRaw, ' ', ' ') }} + jobs: - template: ${{ parameters.jobTemplate }} parameters: @@ -102,38 +128,15 @@ jobs: ${{ each parameter in parameters.jobTemplateParameters }}: ${{ parameter.key }}: ${{ parameter.value }} variables: - - name: _internal - value: ${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }} - - ${{ if eq(variables._internal, true) }}: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: # for private runs, we want to upload perflab data - group: DotNet-HelixApi-Access # perflab upload tokens still exist in this variable group - group: dotnet-benchview + - ${{ if parameters.downloadPdn }}: + - name: _pdnPath + value: '$(Build.ArtifactStagingDirectory)/PDN.zip' # Create a list of all the parts of the job identifier. Will be joined with '_' to create the job name and ' ' to create the display name. This handles runs from both the performance and runtime repository. - - name: _jobNameRuntimeFlavor - ${{ if and(eq(parameters.isScenario, true), ne(parameters.runtimeFlavor, '')) }}: - value: ${{ parameters.runtimeFlavor }} - ${{ else }}: - value: '' - - name: _jobNameIOSLlvmBuild - ${{ if eq(parameters.iOSLlvmBuild, true) }}: - value: iOSLlvmBuild - ${{ else }}: - value: '' - - name: _jobNameIOSStripSymbols - ${{ if eq(parameters.iOSStripSymbols, true) }}: - value: iOSStripSymbols - ${{ else }}: - value: '' - - name: _jobNameHybridGlobalization - ${{ if eq(parameters.hybridGlobalization, true) }}: - value: hybridGlobalization - ${{ else }}: - value: '' - - name: _jobNameRaw - value: ${{ parameters.osGroup }} ${{ parameters.osSubGroup }} ${{ parameters.osVersion }} ${{ parameters.archType }} ${{ variables._jobNameRuntimeFlavor }} ${{ parameters.runtimeType }} ${{ parameters.codeGenType }} ${{ parameters.runKind }} ${{ parameters.machinePool }} ${{ parameters.logicalMachine }} ${{ parameters.javascriptEngine }} ${{ parameters.pgoRunType }} ${{ parameters.r2rRunType }} ${{ parameters.experimentName }} ${{ variables._jobNameIOSLlvmBuild }} ${{ variables._jobNameIOSStripSymbols }} ${{ variables._jobNameHybridGlobalization }} ${{ parameters.additionalJobIdentifier }} - - name: _jobName - value: ${{ replace(variables._jobNameRaw, ' ', ' ') }} - name: _BuildConfig value: ${{ parameters.buildConfig }} - name: _projectFile @@ -141,11 +144,6 @@ jobs: value: ${{ parameters.projectFile }} ${{ else }}: value: ${{ parameters.performanceRepoDir }}/eng/performance/${{ parameters.projectFileName }} - - name: _pdnPath - ${{ if and(eq(variables._internal, true), eq(parameters.downloadPdn, true)) }}: - value: '$(Build.ArtifactStagingDirectory)/PDN.zip' - ${{ else }}: - value: '' - ${{ if eq(parameters.osGroup, 'windows') }}: - name: Python value: 'py -3' @@ -157,9 +155,8 @@ jobs: steps: - ${{ each step in parameters.steps }}: - ${{ step }} - - ${{ if eq(parameters.downloadPdn, true) }}: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.downloadPdn, true)) }}: - task: AzureCLI@2 - condition: ne(variables._pdnPath, '') displayName: 'Download PDN' inputs: azureSubscription: '.NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa)' @@ -171,8 +168,6 @@ jobs: parameters: performanceRepoDir: ${{ parameters.performanceRepoDir }} runPerformanceJobArgs: - - ${{ if eq(variables._internal, true) }}: - - '--internal' - '--run-kind ${{ parameters.runKind }}' - '--architecture ${{ parameters.archType }}' - '--os-group ${{ parameters.osGroup }}' @@ -240,8 +235,10 @@ jobs: - '--ios-llvm-build' - ${{ if eq(parameters.iOSStripSymbols, true) }}: - '--ios-strip-symbols' - - ${{ if ne(variables._pdnPath, '') }}: - - '--pdn-path $(_pdnPath)' + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - '--internal' + - ${{ if parameters.downloadPdn }}: + - '--pdn-path $(_pdnPath)' - ${{ if ne(parameters.additionalSetupParameters, '') }}: - '${{ parameters.additionalSetupParameters }}' - template: /eng/performance/templates/send-to-helix-step.yml diff --git a/eng/performance/templates/run-scenarios-job.yml b/eng/performance/templates/run-scenarios-job.yml index f47e936c14d..3ccaf4f67ec 100644 --- a/eng/performance/templates/run-scenarios-job.yml +++ b/eng/performance/templates/run-scenarios-job.yml @@ -7,6 +7,6 @@ jobs: - template: /eng/performance/templates/run-performance-job.yml parameters: isScenario: true - downloadPdn: ${{ if and(eq(parameters.osGroup, 'windows'), ne(parameters.archType, 'x86')) }} + downloadPdn: ${{ and(eq(parameters.osGroup, 'windows'), ne(parameters.archType, 'x86')) }} ${{ each parameter in parameters }}: ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file From 104d46b5ac5fea33708d4109cda95cd09f09db08 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 8 Jan 2025 06:06:59 +1000 Subject: [PATCH 14/45] Use only compile time parameters for job names --- eng/performance/gc_jobs.yml | 52 -------------- .../templates/run-performance-job.yml | 68 +++++++++++-------- 2 files changed, 39 insertions(+), 81 deletions(-) delete mode 100644 eng/performance/gc_jobs.yml diff --git a/eng/performance/gc_jobs.yml b/eng/performance/gc_jobs.yml deleted file mode 100644 index 7743518a82f..00000000000 --- a/eng/performance/gc_jobs.yml +++ /dev/null @@ -1,52 +0,0 @@ -parameters: - osGroup: '' # required -- windows | linux | macos - osVersion: '' # required -- OS version - kind: '' # required -- benchmark kind. As of today, only "micro" and "mlnet" benchmarks are supported, we plan to add "scenarios" soon - archType: '' # required -- Architecture. Allowed values: x64, x86, arm, arm64 - pool: '' # required -- name of the Helix pool - queue: '' # required -- name of the Helix queue - container: '' # optional -- id of the container - -jobs: -- template: ../common/templates/jobs/jobs.yml - parameters: - enableTelemetry: false - enablePublishBuildArtifacts: true - helixRepo: dotnet/performance - jobs: - - job: '${{ parameters.osGroup }}_${{ parameters.osVersion }}_${{ parameters.archType }}_${{ parameters.kind }}' - displayName: '${{ parameters.osGroup }} ${{ parameters.osVersion }} ${{ parameters.archType }} ${{ parameters.kind }}' - timeoutInMinutes: 320 - variables: - - name: Python - value: 'py -3' - - group: DotNet-HelixApi-Access - workspace: - clean: all - pool: - name: ${{ parameters.pool }} - container: ${{ parameters.container }} - - steps: - - checkout: self - clean: true - - script: $(Python) scripts/gcinfra_setup.py - displayName: Set up GCPerf, GCPerfSim, and C Tools - - #Your steps here - - # - template: /eng/performance/send-to-helix.yml - # parameters: - # HelixSource: '$(HelixSourcePrefix)/dotnet/performance/$(Build.SourceBranch)' # Need to update this - # HelixType: 'test/performance_$(_BuildConfig)/' #Need to update this - # HelixAccessToken: $(HelixApiAccessToken) - # HelixTargetQueues: ${{ parameters.queue }} - # HelixPreCommands: $(HelixPreCommand) - # Creator: $(Creator) - # Architecture: ${{ parameters.archType }} - # TargetCsproj: ${{ parameters.csproj }} - # WorkItemTimeout: 4:00 # 4 hours - # Python: $(Python) - # ${{ if eq(parameters.osGroup, 'windows') }}: - # WorkItemDirectory: '$(Build.SourcesDirectory)\docs' # Need to update this - # CorrelationPayloadDirectory: '$(Build.SourcesDirectory)\notLocked' # Need to update this diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 73b196de96a..1d7fa19318c 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -54,39 +54,37 @@ parameters: iOSStripSymbols: false # optional -- Whether to strip symbols from the iOS build additionalSetupParameters: '' # optional -- Additional arguments to pass to the script -variables: - - name: _jobNameRuntimeFlavor - ${{ if and(eq(parameters.isScenario, true), ne(parameters.runtimeFlavor, '')) }}: - value: ${{ parameters.runtimeFlavor }} - ${{ else }}: - value: '' - - name: _jobNameIOSLlvmBuild - ${{ if eq(parameters.iOSLlvmBuild, true) }}: - value: iOSLlvmBuild - ${{ else }}: - value: '' - - name: _jobNameIOSStripSymbols - ${{ if eq(parameters.iOSStripSymbols, true) }}: - value: iOSStripSymbols - ${{ else }}: - value: '' - - name: _jobNameHybridGlobalization - ${{ if eq(parameters.hybridGlobalization, true) }}: - value: hybridGlobalization - ${{ else }}: - value: '' - - name: _jobNameRaw - value: ${{ parameters.osGroup }} ${{ parameters.osSubGroup }} ${{ parameters.osVersion }} ${{ parameters.archType }} ${{ variables._jobNameRuntimeFlavor }} ${{ parameters.runtimeType }} ${{ parameters.codeGenType }} ${{ parameters.runKind }} ${{ parameters.machinePool }} ${{ parameters.logicalMachine }} ${{ parameters.javascriptEngine }} ${{ parameters.pgoRunType }} ${{ parameters.r2rRunType }} ${{ parameters.experimentName }} ${{ variables._jobNameIOSLlvmBuild }} ${{ variables._jobNameIOSStripSymbols }} ${{ variables._jobNameHybridGlobalization }} ${{ parameters.additionalJobIdentifier }} - - name: _jobName - value: ${{ replace(variables._jobNameRaw, ' ', ' ') }} - jobs: - template: ${{ parameters.jobTemplate }} parameters: enableTelemetry: ${{ parameters.enableTelemetry }} enablePublishBuildArtifacts: true - name: ${{ replace(variables._jobName, ' ', '_') }} - displayName: Performance ${{ variables._jobName }} + name: run_performance_test_${{ replace( + format("{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}", + parameters.runKind, + coalesce(parameters.runtimeFlavor, "NULL"), + coalesce(parameters.runtimeType, "NULL"), + coalesce(parameters.codeGenType, "NULL"), + coalesce(parameters.additionalJobIdentifier, "NULL"), + parameters.osGroup, + parameters.osSubGroup, + coalesce(parameters.osVersion, "NULL"), + parameters.archType, + coalesce(parameters.machinePool, parameters.logicalMachine) + ), "_NULL", "") }} + displayName: Performance ${{ replace( + format("{0} {1} {2} {3} {4} {5}{6} {7} {8} {9}", + parameters.runKind, + coalesce(parameters.runtimeFlavor, "NULL"), + coalesce(parameters.runtimeType, "NULL"), + coalesce(parameters.codeGenType, "NULL"), + coalesce(parameters.additionalJobIdentifier, "NULL"), + parameters.osGroup, + parameters.osSubGroup, + coalesce(parameters.osVersion, "NULL"), + parameters.archType, + coalesce(parameters.machinePool, parameters.logicalMachine) + ), " NULL", "") }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all @@ -100,7 +98,19 @@ jobs: artifacts: publish: logs: - name: Builds_${{ replace(variables._jobName, ' ', '_') }} + name: Builds_${{ replace( + format("{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}", + parameters.runKind, + coalesce(parameters.runtimeFlavor, "NULL"), + coalesce(parameters.runtimeType, "NULL"), + coalesce(parameters.codeGenType, "NULL"), + coalesce(parameters.additionalJobIdentifier, "NULL"), + parameters.osGroup, + parameters.osSubGroup, + coalesce(parameters.osVersion, "NULL"), + parameters.archType, + coalesce(parameters.machinePool, parameters.logicalMachine) + ), "_NULL", "") }} # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: From 25e252186b79c22b534b6864f52698d96c88308f Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 8 Jan 2025 06:32:20 +1000 Subject: [PATCH 15/45] Try remove new lines --- .../templates/run-performance-job.yml | 42 ++----------------- 1 file changed, 3 insertions(+), 39 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 1d7fa19318c..f7f5f8c3403 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -59,32 +59,8 @@ jobs: parameters: enableTelemetry: ${{ parameters.enableTelemetry }} enablePublishBuildArtifacts: true - name: run_performance_test_${{ replace( - format("{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}", - parameters.runKind, - coalesce(parameters.runtimeFlavor, "NULL"), - coalesce(parameters.runtimeType, "NULL"), - coalesce(parameters.codeGenType, "NULL"), - coalesce(parameters.additionalJobIdentifier, "NULL"), - parameters.osGroup, - parameters.osSubGroup, - coalesce(parameters.osVersion, "NULL"), - parameters.archType, - coalesce(parameters.machinePool, parameters.logicalMachine) - ), "_NULL", "") }} - displayName: Performance ${{ replace( - format("{0} {1} {2} {3} {4} {5}{6} {7} {8} {9}", - parameters.runKind, - coalesce(parameters.runtimeFlavor, "NULL"), - coalesce(parameters.runtimeType, "NULL"), - coalesce(parameters.codeGenType, "NULL"), - coalesce(parameters.additionalJobIdentifier, "NULL"), - parameters.osGroup, - parameters.osSubGroup, - coalesce(parameters.osVersion, "NULL"), - parameters.archType, - coalesce(parameters.machinePool, parameters.logicalMachine) - ), " NULL", "") }} + name: run_performance_test_${{ replace(format("{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}", parameters.runKind, coalesce(parameters.runtimeFlavor, "NULL"), coalesce(parameters.runtimeType, "NULL"), coalesce(parameters.codeGenType, "NULL"), coalesce(parameters.additionalJobIdentifier, "NULL"), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, "NULL"), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), "_NULL", "") }} + displayName: Performance ${{ replace(format("{0} {1} {2} {3} {4} {5}{6} {7} {8} {9}", parameters.runKind, coalesce(parameters.runtimeFlavor, "NULL"), coalesce(parameters.runtimeType, "NULL"), coalesce(parameters.codeGenType, "NULL"), coalesce(parameters.additionalJobIdentifier, "NULL"), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, "NULL"), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), " NULL", "") }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all @@ -98,19 +74,7 @@ jobs: artifacts: publish: logs: - name: Builds_${{ replace( - format("{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}", - parameters.runKind, - coalesce(parameters.runtimeFlavor, "NULL"), - coalesce(parameters.runtimeType, "NULL"), - coalesce(parameters.codeGenType, "NULL"), - coalesce(parameters.additionalJobIdentifier, "NULL"), - parameters.osGroup, - parameters.osSubGroup, - coalesce(parameters.osVersion, "NULL"), - parameters.archType, - coalesce(parameters.machinePool, parameters.logicalMachine) - ), "_NULL", "") }} + name: Builds_${{ replace(format("{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}", parameters.runKind, coalesce(parameters.runtimeFlavor, "NULL"), coalesce(parameters.runtimeType, "NULL"), coalesce(parameters.codeGenType, "NULL"), coalesce(parameters.additionalJobIdentifier, "NULL"), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, "NULL"), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), "_NULL", "") }} # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: From 4230f8c81a4049eaabee0a2548fd4bb633f3c388 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 8 Jan 2025 06:38:55 +1000 Subject: [PATCH 16/45] Use single quotes --- eng/performance/templates/run-performance-job.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index f7f5f8c3403..6c3257007ae 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -59,8 +59,8 @@ jobs: parameters: enableTelemetry: ${{ parameters.enableTelemetry }} enablePublishBuildArtifacts: true - name: run_performance_test_${{ replace(format("{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}", parameters.runKind, coalesce(parameters.runtimeFlavor, "NULL"), coalesce(parameters.runtimeType, "NULL"), coalesce(parameters.codeGenType, "NULL"), coalesce(parameters.additionalJobIdentifier, "NULL"), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, "NULL"), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), "_NULL", "") }} - displayName: Performance ${{ replace(format("{0} {1} {2} {3} {4} {5}{6} {7} {8} {9}", parameters.runKind, coalesce(parameters.runtimeFlavor, "NULL"), coalesce(parameters.runtimeType, "NULL"), coalesce(parameters.codeGenType, "NULL"), coalesce(parameters.additionalJobIdentifier, "NULL"), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, "NULL"), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), " NULL", "") }} + name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeFlavor, 'NULL'), coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} + displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4} {5}{6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeFlavor, 'NULL'), coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), ' NULL', '') }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all @@ -74,7 +74,7 @@ jobs: artifacts: publish: logs: - name: Builds_${{ replace(format("{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}", parameters.runKind, coalesce(parameters.runtimeFlavor, "NULL"), coalesce(parameters.runtimeType, "NULL"), coalesce(parameters.codeGenType, "NULL"), coalesce(parameters.additionalJobIdentifier, "NULL"), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, "NULL"), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), "_NULL", "") }} + name: Builds_${{ replace(format('{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeFlavor, 'NULL'), coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: From ed6c507f68d82540a702a61d5e3495a54145e54d Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 8 Jan 2025 06:49:04 +1000 Subject: [PATCH 17/45] Ensure spaces are converted to underscore --- eng/performance/templates/run-performance-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 6c3257007ae..2ea0e9f68b6 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -59,7 +59,7 @@ jobs: parameters: enableTelemetry: ${{ parameters.enableTelemetry }} enablePublishBuildArtifacts: true - name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeFlavor, 'NULL'), coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} + name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeFlavor, 'NULL'), coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4} {5}{6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeFlavor, 'NULL'), coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), ' NULL', '') }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: @@ -74,7 +74,7 @@ jobs: artifacts: publish: logs: - name: Builds_${{ replace(format('{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeFlavor, 'NULL'), coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} + name: Builds_${{ replace(format('{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeFlavor, 'NULL'), coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: From b18dc9d77a7c6a6f5194710d781aa33b588617f6 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 8 Jan 2025 06:58:42 +1000 Subject: [PATCH 18/45] Change runtime flavor to be additional job identifier --- azure-pipelines.yml | 2 ++ eng/performance/templates/run-performance-job.yml | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dcb5afeb73c..eeaa1fbe035 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -402,6 +402,7 @@ jobs: channels: - 8.0 runtimeFlavor: mono + additionalJobIdentifier: Mono # Maui iOS Native AOT scenario benchmarks - template: /eng/performance/templates/build-machine-matrix.yml @@ -418,6 +419,7 @@ jobs: channels: - 8.0 runtimeFlavor: coreclr + additionalJobIdentifier: CoreCLR ## Maui scenario benchmarks #- template: /eng/performance/templates/build-machine-matrix.yml diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 2ea0e9f68b6..b751674f913 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -59,8 +59,8 @@ jobs: parameters: enableTelemetry: ${{ parameters.enableTelemetry }} enablePublishBuildArtifacts: true - name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeFlavor, 'NULL'), coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} - displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4} {5}{6} {7} {8} {9}', parameters.runKind, coalesce(parameters.runtimeFlavor, 'NULL'), coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), ' NULL', '') }} + name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} + displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), ' NULL', '') }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} workspace: clean: all @@ -74,7 +74,7 @@ jobs: artifacts: publish: logs: - name: Builds_${{ replace(format('{0}_{1}_{2}_{3}_{4}_{5}{6}_{7}_{8}_{9}', parameters.runKind, coalesce(parameters.runtimeFlavor, 'NULL'), coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} + name: Builds_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: From 092fb6c8f3ccffdfecb23886cefa451508bce2b5 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 8 Jan 2025 15:44:31 +1000 Subject: [PATCH 19/45] Fix conditions on arguments for python script --- eng/performance/templates/run-performance-job.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index b751674f913..3c9450a8cac 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -107,7 +107,7 @@ jobs: - group: DotNet-HelixApi-Access # perflab upload tokens still exist in this variable group - group: dotnet-benchview - - ${{ if parameters.downloadPdn }}: + - ${{ if eq(parameters.downloadPdn, true) }}: - name: _pdnPath value: '$(Build.ArtifactStagingDirectory)/PDN.zip' # Create a list of all the parts of the job identifier. Will be joined with '_' to create the job name and ' ' to create the display name. This handles runs from both the performance and runtime repository. @@ -157,7 +157,7 @@ jobs: - ${{ elseif ne(parameters.channel, '') }}: - '--performance-repo-ci' - '--channel ${{ parameters.channel }}' - - ${{ if ne(parameters.affinity, '') }}: + - ${{ if notIn(parameters.affinity, '', '0') }}: - '--affinity ${{ parameters.affinity }}' - ${{ if ne(parameters.targetCsproj, '') }}: - '--target-csproj ${{ parameters.targetCsproj }}' @@ -167,7 +167,7 @@ jobs: - '--is-scenario' - ${{ if ne(length(parameters.runEnvVars), 0) }}: - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - - ${{ if ne(parameters.runtimeFlavor, '') }}: + - ${{ if and(eq(parameters.runKind, 'maui_scenarios_ios'), ne(parameters.runtimeFlavor, '') }}: - '--runtime-flavor ${{ parameters.runtimeFlavor }}' - ${{ if eq(parameters.hybridGlobalization, true) }}: - '--hybrid-globalization' @@ -211,7 +211,7 @@ jobs: - '--ios-strip-symbols' - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - '--internal' - - ${{ if parameters.downloadPdn }}: + - ${{ if eq(parameters.downloadPdn, true) }}: - '--pdn-path $(_pdnPath)' - ${{ if ne(parameters.additionalSetupParameters, '') }}: - '${{ parameters.additionalSetupParameters }}' From 7e959aafdb847f7e09341527a8508abc2a9fcdd4 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 8 Jan 2025 15:45:21 +1000 Subject: [PATCH 20/45] Add missing parenthesis --- eng/performance/templates/run-performance-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 3c9450a8cac..3145a8844e4 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -167,7 +167,7 @@ jobs: - '--is-scenario' - ${{ if ne(length(parameters.runEnvVars), 0) }}: - "--run-env-vars ${{ join(' ', parameters.runEnvVars)}}" - - ${{ if and(eq(parameters.runKind, 'maui_scenarios_ios'), ne(parameters.runtimeFlavor, '') }}: + - ${{ if and(eq(parameters.runKind, 'maui_scenarios_ios'), ne(parameters.runtimeFlavor, '')) }}: - '--runtime-flavor ${{ parameters.runtimeFlavor }}' - ${{ if eq(parameters.hybridGlobalization, true) }}: - '--hybrid-globalization' From e1938e65400d0e858515e5968d1afed12ca831c1 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 8 Jan 2025 17:45:11 +1000 Subject: [PATCH 21/45] Ensure buildConfig is properly capitalised for artifacts --- eng/performance/templates/run-performance-job.yml | 9 ++++++++- eng/performance/templates/runtime-perf-job.yml | 8 ++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 3145a8844e4..0fd288077fb 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -111,8 +111,15 @@ jobs: - name: _pdnPath value: '$(Build.ArtifactStagingDirectory)/PDN.zip' # Create a list of all the parts of the job identifier. Will be joined with '_' to create the job name and ' ' to create the display name. This handles runs from both the performance and runtime repository. + - name: buildConfigUpper + ${{ if eq(parameters.buildConfig, 'debug') }}: + value: 'Debug' + ${{ if eq(parameters.buildConfig, 'release') }}: + value: 'Release' + ${{ if eq(parameters.buildConfig, 'checked') }}: + value: 'Checked' - name: _BuildConfig - value: ${{ parameters.buildConfig }} + value: $(buildConfigUpper) - name: _projectFile ${{ if ne(parameters.projectFile, '') }}: value: ${{ parameters.projectFile }} diff --git a/eng/performance/templates/runtime-perf-job.yml b/eng/performance/templates/runtime-perf-job.yml index 657397bf3ac..d050e9069ae 100644 --- a/eng/performance/templates/runtime-perf-job.yml +++ b/eng/performance/templates/runtime-perf-job.yml @@ -127,8 +127,8 @@ jobs: - template: /eng/performance/templates/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/bin - artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_coreclr$(archiveExtension)' - artifactName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_coreclr' + artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_coreclr$(archiveExtension)' + artifactName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_coreclr' displayName: 'Runtime artifacts' - template: /eng/performance/templates/download-artifact-step.yml @@ -144,8 +144,8 @@ jobs: - template: /eng/performance/templates/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/bin - artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.runtimeType }}$(archiveExtension)' - artifactName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_${{ parameters.runtimeType }}' + artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_${{ parameters.runtimeType }}$(archiveExtension)' + artifactName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_${{ parameters.runtimeType }}' displayName: 'Runtime artifacts' - ${{ elseif eq(parameters.runtimeType, 'AndroidMono')}}: # Download artifacts for Android Testing From f2b0f7860dbd1dda03635b7f1f1e6ec543399c0e Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 8 Jan 2025 19:04:24 +1000 Subject: [PATCH 22/45] Ensure args are properly quoted and ensure custom targetCsproj is passed in --- eng/performance/templates/run-performance-job.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 0fd288077fb..41eb95174c1 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -167,9 +167,9 @@ jobs: - ${{ if notIn(parameters.affinity, '', '0') }}: - '--affinity ${{ parameters.affinity }}' - ${{ if ne(parameters.targetCsproj, '') }}: - - '--target-csproj ${{ parameters.targetCsproj }}' + - '--target-csproj "${{ parameters.targetCsproj }}"' - ${{ if ne(parameters.runCategories, '') }}: - - '--run-categories ${{ parameters.runCategories }}' + - '--run-categories "${{ parameters.runCategories }}"' - ${{ if eq(parameters.isScenario, true) }}: - '--is-scenario' - ${{ if ne(length(parameters.runEnvVars), 0) }}: @@ -184,7 +184,7 @@ jobs: - '--dotnet-version-link $(_DotnetVersionParam)' - ${{ elseif ne(parameters.dotnetVersionLink, '') }}: - '--dotnet-version-link ${{ parameters.dotnetVersionLink }}' - - ${{ if ne(parameters.projectFile, '') }}: + - ${{ if or(ne(parameters.projectFile, ''), ne(parameters.projectFileName, 'helix.proj')) }}: - '--project-file $(_projectFile)' - ${{ if ne(length(parameters.frameworks), 0) }}: - '--framework $(_Framework)' @@ -219,7 +219,7 @@ jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - '--internal' - ${{ if eq(parameters.downloadPdn, true) }}: - - '--pdn-path $(_pdnPath)' + - '--pdn-path "$(_pdnPath)"' - ${{ if ne(parameters.additionalSetupParameters, '') }}: - '${{ parameters.additionalSetupParameters }}' - template: /eng/performance/templates/send-to-helix-step.yml From 5db23b37dc2cb7a6c60f4f8e22bc09b16048d321 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 8 Jan 2025 19:22:12 +1000 Subject: [PATCH 23/45] Fix dotnet version link argument --- eng/performance/templates/run-performance-job.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 41eb95174c1..3afac6e9efd 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -94,7 +94,7 @@ jobs: ${{ each versionPair in parameters.dotnetVersionsLinks }}: ${{ versionPair.key }}_Link: _Channel: ${{ versionPair.key }} - _DotnetVersionParam: ${{ versionPair.value }} + _DotnetVersionParam: --dotnet-version-link ${{ versionPair.value }} ${{ if ne(length(parameters.frameworks), 0) }}: ${{ each framework in parameters.frameworks }}: ${{ framework }}: @@ -161,6 +161,7 @@ jobs: - ${{ if or(ne(length(parameters.channels), 0), ne(length(parameters.dotnetVersionsLinks), 0)) }}: - '--performance-repo-ci' - '--channel $(_Channel)' + - $(_DotnetVersionParam) - ${{ elseif ne(parameters.channel, '') }}: - '--performance-repo-ci' - '--channel ${{ parameters.channel }}' @@ -180,9 +181,7 @@ jobs: - '--hybrid-globalization' - ${{ if ne(parameters.osVersion, '') }}: - '--os-version ${{ parameters.osVersion }}' - - ${{ if ne(length(parameters.dotnetVersionsLinks), 0) }}: - - '--dotnet-version-link $(_DotnetVersionParam)' - - ${{ elseif ne(parameters.dotnetVersionLink, '') }}: + - ${{ if ne(parameters.dotnetVersionLink, '') }}: - '--dotnet-version-link ${{ parameters.dotnetVersionLink }}' - ${{ if or(ne(parameters.projectFile, ''), ne(parameters.projectFileName, 'helix.proj')) }}: - '--project-file $(_projectFile)' From f50d21c259c0c6f51adea2a036ff0a7ad847c208 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Thu, 9 Jan 2025 05:43:37 +1000 Subject: [PATCH 24/45] Fix duplicated logs --- eng/performance/templates/run-performance-job.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 3afac6e9efd..9fa581a1ede 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -58,7 +58,6 @@ jobs: - template: ${{ parameters.jobTemplate }} parameters: enableTelemetry: ${{ parameters.enableTelemetry }} - enablePublishBuildArtifacts: true name: run_performance_test_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} displayName: Performance ${{ replace(format('{0} {1} {2} {3} {4}{5} {6} {7} {8}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(parameters.additionalJobIdentifier, 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), ' NULL', '') }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} @@ -74,7 +73,7 @@ jobs: artifacts: publish: logs: - name: Builds_${{ replace(format('{0}_{1}_{2}_{3}_{4}{5}_{6}_{7}_{8}', parameters.runKind, coalesce(parameters.runtimeType, 'NULL'), coalesce(parameters.codeGenType, 'NULL'), coalesce(replace(parameters.additionalJobIdentifier, ' ', '_'), 'NULL'), parameters.osGroup, parameters.osSubGroup, coalesce(parameters.osVersion, 'NULL'), parameters.archType, coalesce(parameters.machinePool, parameters.logicalMachine)), '_NULL', '') }} + name: Logs_$[replace(variables['Agent.JobName'], ' ', '_')]$[replace(format('_Attempt{0}', variables['System.JobAttempt']), '_Attemp1', '')] # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: From 999bba6d1b884b107c9293f0242b8dcb8b030e88 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Thu, 9 Jan 2025 09:01:15 +1000 Subject: [PATCH 25/45] Fix logs name --- eng/performance/templates/run-performance-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 9fa581a1ede..d7356a53065 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -73,7 +73,7 @@ jobs: artifacts: publish: logs: - name: Logs_$[replace(variables['Agent.JobName'], ' ', '_')]$[replace(format('_Attempt{0}', variables['System.JobAttempt']), '_Attemp1', '')] + name: $[replace(replace(format('Logs_{0}_Attempt{1}', variables['Agent.JobName'], variables['System.JobAttempt']), '_Attemp1', ''), ' ', '_')] # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: From 13983ff3aec53a12175893cf766d9be6c2ca5e07 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Thu, 9 Jan 2025 09:03:43 +1000 Subject: [PATCH 26/45] Fix attempt# replace --- eng/performance/templates/run-performance-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index d7356a53065..8270c219c7a 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -73,7 +73,7 @@ jobs: artifacts: publish: logs: - name: $[replace(replace(format('Logs_{0}_Attempt{1}', variables['Agent.JobName'], variables['System.JobAttempt']), '_Attemp1', ''), ' ', '_')] + name: $[replace(replace(format('Logs_{0}_Attempt{1}', variables['Agent.JobName'], variables['System.JobAttempt']), '_Attempt1', ''), ' ', '_')] # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: From 3be0ce3e238b506fab5fea1b6190b54531c48ca6 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Thu, 9 Jan 2025 10:33:24 +1000 Subject: [PATCH 27/45] Another attempt at getting the log name to work --- eng/performance/templates/run-performance-job.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 8270c219c7a..3cb9ff68c72 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -73,7 +73,7 @@ jobs: artifacts: publish: logs: - name: $[replace(replace(format('Logs_{0}_Attempt{1}', variables['Agent.JobName'], variables['System.JobAttempt']), '_Attempt1', ''), ' ', '_')] + name: $(_logName) # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: @@ -130,6 +130,8 @@ jobs: - ${{ if ne(parameters.osGroup, 'windows') }}: - name: Python value: 'python3' + - name: _logName + value: $[replace(replace(format('Logs_{0}_Attempt{1}', variables['Agent.JobName'], variables['System.JobAttempt']), '_Attempt1', ''), ' ', '_')] - ${{ each variable in parameters.variables }}: - ${{ variable }} steps: From e37c149140abbba3595e01488c04f0a8a638e67f Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Thu, 9 Jan 2025 11:20:01 +1000 Subject: [PATCH 28/45] Remove runtime expression from logs name --- eng/performance/templates/run-performance-job.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 3cb9ff68c72..ab8aa6fe25a 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -73,7 +73,7 @@ jobs: artifacts: publish: logs: - name: $(_logName) + name: Logs_$(Agent.JobName)_Attempt_$(System.JobAttempt) # Component governance does not work on musl machines ${{ if eq(parameters.osSubGroup, '_musl') }}: @@ -130,8 +130,6 @@ jobs: - ${{ if ne(parameters.osGroup, 'windows') }}: - name: Python value: 'python3' - - name: _logName - value: $[replace(replace(format('Logs_{0}_Attempt{1}', variables['Agent.JobName'], variables['System.JobAttempt']), '_Attempt1', ''), ' ', '_')] - ${{ each variable in parameters.variables }}: - ${{ variable }} steps: From 7a161ca2873eb3ff1e83c63a4885ad69b386dfb8 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Thu, 9 Jan 2025 11:36:09 +1000 Subject: [PATCH 29/45] Ensure sys.exit is called in run_performance_job.py --- scripts/run_performance_job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index b20b1e2709d..4ea888667e1 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -1202,4 +1202,5 @@ def main(argv: List[str]): return 1 if __name__ == "__main__": - main(sys.argv) + return_code = main(sys.argv) + sys.exit(return_code) From ee6b3bdc183d262c4db9647a36088dde591ff1a3 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 13:15:08 +1000 Subject: [PATCH 30/45] Move all runtime jobs to performance repository --- .../templates/run-performance-job.yml | 26 +- .../runtime-ios-scenarios-perf-jobs.yml | 140 ++++++++++ .../templates/runtime-perf-job.yml | 20 +- .../templates/runtime-perf-jobs.yml | 263 ++++++++++++++++++ .../templates/runtime-perf-slow-jobs.yml | 249 +++++++++++++++++ .../templates/runtime-wasm-perf-jobs.yml | 131 +++++++++ 6 files changed, 813 insertions(+), 16 deletions(-) create mode 100644 eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml create mode 100644 eng/performance/templates/runtime-perf-jobs.yml create mode 100644 eng/performance/templates/runtime-perf-slow-jobs.yml create mode 100644 eng/performance/templates/runtime-wasm-perf-jobs.yml diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index ab8aa6fe25a..be69aa5edc5 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -18,6 +18,10 @@ parameters: continueOnError: false # optional -- Whether to continue on error buildConfig: '' # optional -- Build configuration variables: [] # optional -- Additional variables to set for the job + performanceRepoAlias: self # optional -- The alias for the performance repository + runtimeRepoAlias: runtime # optional -- The alias for the runtime repository + isRuntimeJob: false # optional -- Whether the performance job runs against a build of the runtime repository + checkoutFetchDepth: 20 # optional -- Fetch depth for the checkout # Passed into run performance job script runKind: '' # required -- micro, scenarios, blazor_scenarios ... @@ -39,8 +43,6 @@ parameters: projectFileName: 'helix.proj' # optional -- Name of Helix project file in /eng/performance directory projectFile: '' # optional -- Full path to project file framework: '' # optional -- Framework to run against (use frameworks). Ignored if frameworks is non-empty. - performanceRepoDir: $(Build.SourcesDirectory) # optional -- Path to the performance repo - runtimeRepoDir: '' # optional -- Path to the runtime repo codeGenType: '' # optional -- CodeGen type (e.g. JIT, AOT) runtimeType: '' # optional -- Runtime type (e.g. mono, wasm, coreclr) useLocalCommitTime: false # optional -- Whether to use the local commit time or fetch it from the remote repository @@ -123,7 +125,7 @@ jobs: ${{ if ne(parameters.projectFile, '') }}: value: ${{ parameters.projectFile }} ${{ else }}: - value: ${{ parameters.performanceRepoDir }}/eng/performance/${{ parameters.projectFileName }} + value: $(Build.SourcesDirectory)/performance/eng/performance/${{ parameters.projectFileName }} - ${{ if eq(parameters.osGroup, 'windows') }}: - name: Python value: 'py -3' @@ -133,6 +135,15 @@ jobs: - ${{ each variable in parameters.variables }}: - ${{ variable }} steps: + - checkout: ${{ parameters.performanceRepoAlias }} + path: s/performance + clean: true + fetchDepth: ${{ parameters.checkoutFetchDepth }} + - ${{ if parameters.isRuntimeJob }}: + - checkout: ${{ parameters.runtimeRepoAlias }} + path: s + clean: true + fetchDepth: ${{ parameters.checkoutFetchDepth }} - ${{ each step in parameters.steps }}: - ${{ step }} - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.downloadPdn, true)) }}: @@ -146,7 +157,7 @@ jobs: az storage blob download --auth-mode login --account-name pvscmdupload --container-name assets --name paint.net.5.0.3.portable.${{ parameters.archType }}.zip --file $(_pdnPath) - template: /eng/performance/templates/run-performance-job-script-step.yml parameters: - performanceRepoDir: ${{ parameters.performanceRepoDir }} + performanceRepoDir: $(Build.SourcesDirectory)/performance runPerformanceJobArgs: - '--run-kind ${{ parameters.runKind }}' - '--architecture ${{ parameters.archType }}' @@ -188,10 +199,9 @@ jobs: - '--framework $(_Framework)' - ${{ elseif ne(parameters.framework, '') }}: - '--framework ${{ parameters.framework }}' - - ${{ if ne(parameters.performanceRepoDir, '.') }}: - - '--performance-repo-dir ${{ parameters.performanceRepoDir }}' - - ${{ if ne(parameters.runtimeRepoDir, '') }}: - - '--runtime-repo-dir ${{ parameters.runtimeRepoDir }}' + - '--performance-repo-dir $(Build.SourcesDirectory)/performance' + - ${{ if parameters.isRuntimeJob }}: + - '--runtime-repo-dir $(Build.SourcesDirectory)' - ${{ if ne(parameters.codeGenType, '') }}: - '--codegen-type ${{ parameters.codeGenType }}' - ${{ if ne(parameters.runtimeType, '') }}: diff --git a/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml b/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml new file mode 100644 index 00000000000..21b335a5530 --- /dev/null +++ b/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml @@ -0,0 +1,140 @@ +parameters: + hybridGlobalization: True + runtimeRepoAlias: runtime + performanceRepoAlias: self + jobParameters: {} + +jobs: + - template: /eng/pipelines/coreclr/templates/perf-ios-scenarios-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + parameters: + hybridGlobalization: ${{ parameters.hybridGlobalization }} + + # run mono iOS scenarios scenarios HybridGlobalization + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - osx_x64 + jobParameters: + runtimeType: iOSMono + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj + runKind: ios_scenarios + isScenario: true + logicalMachine: 'perfiphone12mini' + iOSLlvmBuild: False + iOSStripSymbols: False + hybridGlobalization: ${{ parameters.hybridGlobalization }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - osx_x64 + jobParameters: + runtimeType: iOSMono + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj + runKind: ios_scenarios + isScenario: true + logicalMachine: 'perfiphone12mini' + iOSLlvmBuild: False + iOSStripSymbols: True + hybridGlobalization: ${{ parameters.hybridGlobalization }} + additionalJobIdentifier: iOSStripSymbols + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - osx_x64 + jobParameters: + runtimeType: iOSMono + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj + runKind: ios_scenarios + isScenario: true + logicalMachine: 'perfiphone12mini' + iOSLlvmBuild: True + iOSStripSymbols: False + hybridGlobalization: ${{ parameters.hybridGlobalization }} + additionalJobIdentifier: iOSLlvmBuild + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - osx_x64 + jobParameters: + runtimeType: iOSMono + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj + runKind: ios_scenarios + isScenario: true + logicalMachine: 'perfiphone12mini' + iOSLlvmBuild: True + iOSStripSymbols: True + hybridGlobalization: ${{ parameters.hybridGlobalization }} + additionalJobIdentifier: iOSLlvmBuild iOSStripSymbols + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run NativeAOT iOS scenarios HybridGlobalization + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - osx_x64 + jobParameters: + runtimeType: iOSNativeAOT + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj + runKind: ios_scenarios + isScenario: true + logicalMachine: 'perfiphone12mini' + iOSStripSymbols: False + hybridGlobalization: ${{ parameters.hybridGlobalization }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - osx_x64 + jobParameters: + runtimeType: iOSNativeAOT + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj + runKind: ios_scenarios + isScenario: true + logicalMachine: 'perfiphone12mini' + iOSStripSymbols: True + hybridGlobalization: ${{ parameters.hybridGlobalization }} + additionalJobIdentifier: iOSStripSymbols + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/performance/templates/runtime-perf-job.yml b/eng/performance/templates/runtime-perf-job.yml index d050e9069ae..1249cfff30f 100644 --- a/eng/performance/templates/runtime-perf-job.yml +++ b/eng/performance/templates/runtime-perf-job.yml @@ -16,8 +16,9 @@ parameters: isScenario: false downloadSpecificBuild: null # buildId, pipeline, branchName, project crossBuild: false - runtimeRepoDir: '' - performanceRepoDir: '' + runtimeRepoAlias: runtime + performanceRepoAlias: self + selfIsRuntime: true # Please see run-performance-job.yml for additional parameters that can be specified as they will be forwarded through jobs: @@ -27,6 +28,7 @@ jobs: ${{ if notIn(parameter.key, 'steps', 'variables', 'framework') }}: ${{ parameter.key }}: ${{ parameter.value }} + isRuntimeJob: true useLocalCommitTime: true frameworks: - ${{ parameters.framework }} @@ -57,9 +59,9 @@ jobs: - name: _crossBuildPropertyArg value: '-cross' - name: librariesDownloadDir - value: ${{ parameters.runtimeRepoDir }}/artifacts + value: $(Build.SourcesDirectory)/artifacts - name: builtAppDir - value: ${{ parameters.runtimeRepoDir }} + value: $(Build.SourcesDirectory) - name: librariesOverrideArg value: '' - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: @@ -91,8 +93,10 @@ jobs: steps: - ${{ each step in parameters.steps }}: - ${{ step }} + - ${{ if eq(parameters.osGroup, 'windows') }}: + - template: /eng/pipelines/common/templates/disable-vsupdate-or-failfast.yml@${{ parameters.runtimeRepoAlias }} - script: | - ${{ parameters.runtimeRepoDir }}/eng/common/msbuild.sh ${{ parameters.runtimeRepoDir }}/eng/testing/performance/add_properties_to_pipeline.proj /t:SetVariables + $(Build.SourcesDirectory)/eng/common/msbuild.sh $(Build.SourcesDirectory)/eng/testing/performance/add_properties_to_pipeline.proj /t:SetVariables displayName: Add Properties To Pipeline Env - ${{ if eq(parameters.runtimeType, 'wasm') }}: @@ -212,20 +216,20 @@ jobs: - ${{ if notIn(parameters.runtimeType, 'wasm', 'AndroidMono', 'iOSMono', 'iOSNativeAOT') }}: - ${{ if ne(parameters.runtimeFlavor, 'Mono') }}: # Create Core_Root - - script: ${{ parameters.runtimeRepoDir }}/src/tests/build$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} generatelayoutonly $(librariesOverrideArg) $(_crossBuildPropertyArg) + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) ${{ parameters.buildConfig }} ${{ parameters.archType }} generatelayoutonly $(librariesOverrideArg) $(_crossBuildPropertyArg) displayName: Create Core_Root - ${{ elseif ne(parameters.osGroup, 'windows') }}: - task: CopyFiles@2 displayName: Create .dotnet-mono folder inputs: SourceFolder: $(librariesDownloadDir)/bin/testhost/${{ parameters.framework }}-${{ parameters.osGroup }}-$(buildConfigUpper)-${{ parameters.archType }} - TargetFolder: ${{ parameters.runtimeRepoDir }}/.dotnet-mono + TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono - task: CopyFiles@2 displayName: Copy corerun to .dotnet-mono inputs: SourceFolder: $(librariesDownloadDir)/bin/coreclr/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper) Contents: 'corerun*' - TargetFolder: ${{ parameters.runtimeRepoDir }}/.dotnet-mono/shared/Microsoft.NETCore.App/$(productVersion) + TargetFolder: $(Build.SourcesDirectory)/.dotnet-mono/shared/Microsoft.NETCore.App/$(productVersion) - ${{ if and(eq(parameters.isScenario, true), eq(parameters.runtimeType, 'wasm')) }}: - script: >- diff --git a/eng/performance/templates/runtime-perf-jobs.yml b/eng/performance/templates/runtime-perf-jobs.yml new file mode 100644 index 00000000000..7ae3068fe3f --- /dev/null +++ b/eng/performance/templates/runtime-perf-jobs.yml @@ -0,0 +1,263 @@ +parameters: + perfBranch: 'main' # Currently only used for building the BDN App + runtimeRepoAlias: runtime + performanceRepoAlias: self + jobParameters: {} + +jobs: + - template: /eng/pipelines/coreclr/templates/perf-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + parameters: + perfBranch: ${{ parameters.perfBranch }} + + # Build and run iOS Mono and NativeAOT scenarios + - template: /eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml + parameters: + hybridGlobalization: True + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + jobParameters: ${{ parameters.jobParameters }} + + # run android scenarios + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - windows_x64 + jobParameters: + runtimeType: AndroidMono + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/android_scenarios.proj + runKind: android_scenarios + isScenario: true + logicalMachine: 'perfpixel4a' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run mono microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: mono + runKind: micro_mono + logicalMachine: 'perftiger' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run mono interpreter perf job + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: mono + codeGenType: 'Interpreter' + runKind: micro_mono + logicalMachine: 'perftiger' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run mono aot microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + buildConfig: release + runtimeFlavor: aot + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: mono + codeGenType: 'AOT' + runKind: micro_mono + logicalMachine: 'perftiger' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr perftiger microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + - windows_x64 + - windows_x86 + - linux_musl_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perftiger' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr perftiger microbenchmarks no dynamic pgo perf jobs + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perftiger' + pgoRunType: nodynamicpgo + additionalJobIdentifier: nodynamicpgo + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perftiger' + pgoRunType: nodynamicpgo + additionalJobIdentifier: nodynamicpgo + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr perftiger microbenchmarks no R2R perf jobs + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perftiger' + r2rRunType: nor2r + additionalJobIdentifier: nor2r + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perftiger' + r2rRunType: nor2r + additionalJobIdentifier: nor2r + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr perfowl microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + - windows_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfowl' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr perfviper microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + - windows_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfviper' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr perfviper microbenchmarks perf jitoptrepeat jobs + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_x64 + - windows_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfviper' + experimentName: 'jitoptrepeat' + additionalJobIdentifier: jitoptrepeat + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr crossgen perf job + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + - windows_x86 + jobParameters: + liveLibrariesBuildConfig: Release + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj + runKind: crossgen_scenarios + isScenario: true + logicalMachine: 'perftiger_crossgen' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/performance/templates/runtime-perf-slow-jobs.yml b/eng/performance/templates/runtime-perf-slow-jobs.yml new file mode 100644 index 00000000000..c58ff460645 --- /dev/null +++ b/eng/performance/templates/runtime-perf-slow-jobs.yml @@ -0,0 +1,249 @@ +parameters: + runScheduledJobs: false + runPrivateJobs: false + runtimeRepoAlias: runtime + performanceRepoAlias: self + jobParameters: {} + +jobs: + - ${{ if parameters.runScheduledJobs }}: + - template: /eng/pipelines/coreclr/templates/perf-arm64-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + parameters: + mono: true + + # run arm64 interpreter jobs for mono + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: mono + codeGenType: 'Interpreter' + runKind: micro_mono + logicalMachine: 'perfampere' + timeoutInMinutes: 720 + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run arm64 jit jobs for mono + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: mono + runKind: micro_mono + logicalMachine: 'perfampere' + timeoutInMinutes: 720 + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - ${{ if parameters.runPrivateJobs }}: + - template: /eng/pipelines/coreclr/templates/perf-arm64-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + parameters: + coreclr: true + monoAot: true + + # run mono aot microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + buildConfig: release + runtimeFlavor: aot + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: mono + codeGenType: 'AOT' + runKind: micro_mono + logicalMachine: 'perfampere' + timeoutInMinutes: 780 + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr Linux arm64 ampere microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfampere' + timeoutInMinutes: 780 + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + #run coreclr Linux arm64 ampere no dynamic pgo microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfampere' + timeoutInMinutes: 780 + pgoRunType: nodynamicpgo + additionalJobIdentifier: nodynamicpgo + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + #run coreclr Linux arm64 ampere no R2R microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfampere' + timeoutInMinutes: 780 + r2rRunType: nor2r + additionalJobIdentifier: nor2r + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr Windows arm64 microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfsurf' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr Windows arm64 ampere microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfampere' + timeoutInMinutes: 780 + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr Windows arm64 ampere no dynamic pgo microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfampere' + pgoRunType: nodynamicpgo + timeoutInMinutes: 780 + additionalJobIdentifier: nodynamicpgo + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run coreclr Windows arm64 ampere no R2R microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + runKind: micro + logicalMachine: 'perfampere' + r2rRunType: nor2r + timeoutInMinutes: 780 + additionalJobIdentifier: nor2r + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Uncomment once we fix https://github.com/dotnet/performance/issues/1950 + # # run coreclr linux crossgen perf job + # - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + # parameters: + # jobTemplate: /eng/performance/templates/runtime-perf-job.yml + # buildConfig: release + # runtimeFlavor: coreclr + # platforms: + # - linux_arm64 + # jobParameters: + # liveLibrariesBuildConfig: Release + # projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj + # runKind: crossgen_scenarios + # isScenario: true + # logicalMachine: 'perfa64' + # runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + # performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} + + # # run coreclr windows crossgen perf job + # - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + # parameters: + # jobTemplate: /eng/performance/templates/runtime-perf-job.yml + # buildConfig: release + # runtimeFlavor: coreclr + # platforms: + # - windows_arm64 + # jobParameters: + # liveLibrariesBuildConfig: Release + # projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj + # runKind: crossgen_scenarios + # isScenario: true + # logicalMachine: 'perfsurf' + # runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + # performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + # ${{ each parameter in parameters.jobParameters }}: + # ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/performance/templates/runtime-wasm-perf-jobs.yml b/eng/performance/templates/runtime-wasm-perf-jobs.yml new file mode 100644 index 00000000000..5d9960f77b6 --- /dev/null +++ b/eng/performance/templates/runtime-wasm-perf-jobs.yml @@ -0,0 +1,131 @@ +# This contains all the wasm related perf jobs +parameters: + runProfile: 'non-v8' + downloadSpecificBuild: null # { buildId, pipeline, branchName, project } + runtimeRepoAlias: runtime + performanceRepoAlias: self + jobParameters: {} + +jobs: +- ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: + # build mono on wasm - if not using an existing build + - template: /eng/pipelines/coreclr/templates/perf-wasm-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + +- ${{ if eq(parameters.runProfile, 'non-v8') }}: + #run mono wasm microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + buildConfig: Release + runtimeFlavor: wasm + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: wasm + codeGenType: 'wasm' + runKind: micro + logicalMachine: 'perftiger' + javascriptEngine: 'javascriptcore' + additionalJobIdentifier: javascriptcore + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + #run mono wasm aot microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? + buildconfig: Release + runtimeflavor: wasm + platforms: + - linux_x64 + jobparameters: + livelibrariesbuildconfig: Release + runtimetype: wasm + codegentype: 'aot' + runkind: micro + logicalMachine: 'perftiger' + javascriptengine: 'javascriptcore' + additionalJobIdentifier: javascriptcore + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +- ${{ if eq(parameters.runProfile, 'v8') }}: + # run mono wasm interpreter (default) microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + buildConfig: release + runtimeFlavor: wasm + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: wasm + codeGenType: 'wasm' + runKind: micro + logicalMachine: 'perftiger' + javascriptEngine: 'v8' + # For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm) + #additionalSetupParameters: '--dotnet-versions 8.0.0' + additionalJobIdentifier: v8 + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + #run mono wasm aot microbenchmarks perf job + # Disabled for runtime-wasm-perf on PRs due to https://github.com/dotnet/runtime/issues/95101 + - ${{if not(in(variables['Build.DefinitionName'], 'runtime-wasm-perf')) }}: + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? + buildconfig: release + runtimeflavor: wasm + platforms: + - linux_x64 + jobparameters: + livelibrariesbuildconfig: Release + runtimetype: wasm + codegentype: 'aot' + runkind: micro + logicalMachine: 'perftiger' + javascriptEngine: 'v8' + # For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm) + #additionalSetupParameters: '--dotnet-versions 8.0.0' # passed to ci_setup.py + additionalJobIdentifier: v8 + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # run mono wasm blazor perf job + - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/performance/templates/runtime-perf-job.yml + buildConfig: release + runtimeFlavor: wasm + platforms: + - linux_x64 + jobParameters: + liveLibrariesBuildConfig: Release + runtimeType: wasm + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/blazor_perf.proj + runKind: blazor_scenarios + isScenario: true + # For working with a newer sdk, and previous tfm (eg. 9.0 sdk, and net8.0 tfm) + #additionalSetupParameters: '--dotnetversions 8.0.0' # passed to run-performance-job.py + logicalMachine: 'perftiger' + downloadSpecificBuild: ${{ parameters.downloadSpecificBuild }} + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file From 774035b31d9759b2dff51ef4e4c6c4c0be57a9b8 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 13:17:13 +1000 Subject: [PATCH 31/45] Fix cross repository template reference --- .../runtime-ios-scenarios-perf-jobs.yml | 12 ++++----- .../templates/runtime-perf-jobs.yml | 26 +++++++++---------- .../templates/runtime-wasm-perf-jobs.yml | 6 ++--- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml b/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml index 21b335a5530..8f78e4b1cd7 100644 --- a/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml +++ b/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml @@ -10,7 +10,7 @@ jobs: hybridGlobalization: ${{ parameters.hybridGlobalization }} # run mono iOS scenarios scenarios HybridGlobalization - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -31,7 +31,7 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -53,7 +53,7 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -75,7 +75,7 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -98,7 +98,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # run NativeAOT iOS scenarios HybridGlobalization - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -118,7 +118,7 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release diff --git a/eng/performance/templates/runtime-perf-jobs.yml b/eng/performance/templates/runtime-perf-jobs.yml index 7ae3068fe3f..095d7726932 100644 --- a/eng/performance/templates/runtime-perf-jobs.yml +++ b/eng/performance/templates/runtime-perf-jobs.yml @@ -18,7 +18,7 @@ jobs: jobParameters: ${{ parameters.jobParameters }} # run android scenarios - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -37,7 +37,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # run mono microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -55,7 +55,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # run mono interpreter perf job - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -74,7 +74,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # run mono aot microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? buildConfig: release @@ -93,7 +93,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # run coreclr perftiger microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -113,7 +113,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # run coreclr perftiger microbenchmarks no dynamic pgo perf jobs - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -131,7 +131,7 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -150,7 +150,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # run coreclr perftiger microbenchmarks no R2R perf jobs - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -168,7 +168,7 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -187,7 +187,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # run coreclr perfowl microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -205,7 +205,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # run coreclr perfviper microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -223,7 +223,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # run coreclr perfviper microbenchmarks perf jitoptrepeat jobs - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release @@ -243,7 +243,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # run coreclr crossgen perf job - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release diff --git a/eng/performance/templates/runtime-wasm-perf-jobs.yml b/eng/performance/templates/runtime-wasm-perf-jobs.yml index 5d9960f77b6..f8cd786ccf1 100644 --- a/eng/performance/templates/runtime-wasm-perf-jobs.yml +++ b/eng/performance/templates/runtime-wasm-perf-jobs.yml @@ -58,7 +58,7 @@ jobs: - ${{ if eq(parameters.runProfile, 'v8') }}: # run mono wasm interpreter (default) microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? buildConfig: release @@ -84,7 +84,7 @@ jobs: #run mono wasm aot microbenchmarks perf job # Disabled for runtime-wasm-perf on PRs due to https://github.com/dotnet/runtime/issues/95101 - ${{if not(in(variables['Build.DefinitionName'], 'runtime-wasm-perf')) }}: - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? buildconfig: release @@ -108,7 +108,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # run mono wasm blazor perf job - - template: /eng/pipelines/common/platform-matrix.yml${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/performance/templates/runtime-perf-job.yml buildConfig: release From e13a3458b23e6d7c181c47c6770d916a5d607c74 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 13:19:50 +1000 Subject: [PATCH 32/45] Ensure runtime-perf-job.yml uses repository reference --- .../runtime-ios-scenarios-perf-jobs.yml | 12 ++++----- .../templates/runtime-perf-jobs.yml | 26 +++++++++---------- .../templates/runtime-perf-slow-jobs.yml | 24 ++++++++--------- .../templates/runtime-wasm-perf-jobs.yml | 10 +++---- 4 files changed, 36 insertions(+), 36 deletions(-) diff --git a/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml b/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml index 8f78e4b1cd7..858ddfb8e35 100644 --- a/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml +++ b/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml @@ -12,7 +12,7 @@ jobs: # run mono iOS scenarios scenarios HybridGlobalization - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -33,7 +33,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -55,7 +55,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -77,7 +77,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -100,7 +100,7 @@ jobs: # run NativeAOT iOS scenarios HybridGlobalization - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -120,7 +120,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: diff --git a/eng/performance/templates/runtime-perf-jobs.yml b/eng/performance/templates/runtime-perf-jobs.yml index 095d7726932..6773d1dc4a4 100644 --- a/eng/performance/templates/runtime-perf-jobs.yml +++ b/eng/performance/templates/runtime-perf-jobs.yml @@ -20,7 +20,7 @@ jobs: # run android scenarios - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -39,7 +39,7 @@ jobs: # run mono microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -57,7 +57,7 @@ jobs: # run mono interpreter perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -76,7 +76,7 @@ jobs: # run mono aot microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? buildConfig: release runtimeFlavor: aot platforms: @@ -95,7 +95,7 @@ jobs: # run coreclr perftiger microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -115,7 +115,7 @@ jobs: # run coreclr perftiger microbenchmarks no dynamic pgo perf jobs - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -133,7 +133,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -152,7 +152,7 @@ jobs: # run coreclr perftiger microbenchmarks no R2R perf jobs - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -170,7 +170,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -189,7 +189,7 @@ jobs: # run coreclr perfowl microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -207,7 +207,7 @@ jobs: # run coreclr perfviper microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -225,7 +225,7 @@ jobs: # run coreclr perfviper microbenchmarks perf jitoptrepeat jobs - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -245,7 +245,7 @@ jobs: # run coreclr crossgen perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: diff --git a/eng/performance/templates/runtime-perf-slow-jobs.yml b/eng/performance/templates/runtime-perf-slow-jobs.yml index c58ff460645..9ccee741692 100644 --- a/eng/performance/templates/runtime-perf-slow-jobs.yml +++ b/eng/performance/templates/runtime-perf-slow-jobs.yml @@ -14,7 +14,7 @@ jobs: # run arm64 interpreter jobs for mono - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -34,7 +34,7 @@ jobs: # run arm64 jit jobs for mono - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -59,7 +59,7 @@ jobs: # run mono aot microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? buildConfig: release runtimeFlavor: aot platforms: @@ -79,7 +79,7 @@ jobs: # run coreclr Linux arm64 ampere microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -97,7 +97,7 @@ jobs: #run coreclr Linux arm64 ampere no dynamic pgo microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -117,7 +117,7 @@ jobs: #run coreclr Linux arm64 ampere no R2R microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -137,7 +137,7 @@ jobs: # run coreclr Windows arm64 microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -154,7 +154,7 @@ jobs: # run coreclr Windows arm64 ampere microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -172,7 +172,7 @@ jobs: # run coreclr Windows arm64 ampere no dynamic pgo microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -192,7 +192,7 @@ jobs: # run coreclr Windows arm64 ampere no R2R microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -213,7 +213,7 @@ jobs: # # run coreclr linux crossgen perf job # - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} # parameters: - # jobTemplate: /eng/performance/templates/runtime-perf-job.yml + # jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # buildConfig: release # runtimeFlavor: coreclr # platforms: @@ -232,7 +232,7 @@ jobs: # # run coreclr windows crossgen perf job # - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} # parameters: - # jobTemplate: /eng/performance/templates/runtime-perf-job.yml + # jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # buildConfig: release # runtimeFlavor: coreclr # platforms: diff --git a/eng/performance/templates/runtime-wasm-perf-jobs.yml b/eng/performance/templates/runtime-wasm-perf-jobs.yml index f8cd786ccf1..49d2df6450f 100644 --- a/eng/performance/templates/runtime-wasm-perf-jobs.yml +++ b/eng/performance/templates/runtime-wasm-perf-jobs.yml @@ -15,7 +15,7 @@ jobs: #run mono wasm microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? buildConfig: Release runtimeFlavor: wasm platforms: @@ -37,7 +37,7 @@ jobs: #run mono wasm aot microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # note: should we move this file out of coreclr tempelates because it contains mono jobs? buildconfig: Release runtimeflavor: wasm platforms: @@ -60,7 +60,7 @@ jobs: # run mono wasm interpreter (default) microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: wasm platforms: @@ -86,7 +86,7 @@ jobs: - ${{if not(in(variables['Build.DefinitionName'], 'runtime-wasm-perf')) }}: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # note: should we move this file out of coreclr tempelates because it contains mono jobs? buildconfig: release runtimeflavor: wasm platforms: @@ -110,7 +110,7 @@ jobs: # run mono wasm blazor perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml + jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: wasm platforms: From 3e9d63308a9e88ad8bc40a61a7be1b9790064e8b Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 14:29:50 +1000 Subject: [PATCH 33/45] Ensure runtime is cloned first as it cleans the directory --- eng/performance/templates/run-performance-job.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index be69aa5edc5..e7b68d644a4 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -135,15 +135,15 @@ jobs: - ${{ each variable in parameters.variables }}: - ${{ variable }} steps: - - checkout: ${{ parameters.performanceRepoAlias }} - path: s/performance - clean: true - fetchDepth: ${{ parameters.checkoutFetchDepth }} - ${{ if parameters.isRuntimeJob }}: - checkout: ${{ parameters.runtimeRepoAlias }} path: s clean: true fetchDepth: ${{ parameters.checkoutFetchDepth }} + - checkout: ${{ parameters.performanceRepoAlias }} + path: s/performance + clean: true + fetchDepth: ${{ parameters.checkoutFetchDepth }} - ${{ each step in parameters.steps }}: - ${{ step }} - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.downloadPdn, true)) }}: From c7cb12d1a095d87eb4f8d4230b1e7a71db5b2454 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 16:52:43 +1000 Subject: [PATCH 34/45] Port changes from dotnet/runtime --- .../templates/runtime-perf-jobs.yml | 91 ++++++++++++++++++- 1 file changed, 87 insertions(+), 4 deletions(-) diff --git a/eng/performance/templates/runtime-perf-jobs.yml b/eng/performance/templates/runtime-perf-jobs.yml index 6773d1dc4a4..34d2365f08d 100644 --- a/eng/performance/templates/runtime-perf-jobs.yml +++ b/eng/performance/templates/runtime-perf-jobs.yml @@ -1,8 +1,91 @@ parameters: - perfBranch: 'main' # Currently only used for building the BDN App - runtimeRepoAlias: runtime - performanceRepoAlias: self - jobParameters: {} + - name: perfBranch # Currently only used for building the BDN App + type: string + default: main + - name: jobParameters + type: object + default: {} + - name: runtimeRepoAlias + type: string + default: runtime + - name: performanceRepoAlias + type: string + default: self + - name: crossgen + type: object + default: + enabled: true + configs: + - windows_x64 + - windows_x86 + - name: viperJitOptRepeat + type: object + default: + enabled: true + configs: + - windows_x64 + - linux_x64 + - name: viperMicro + type: object + default: + enabled: true + configs: + - windows_x64 + - linux_x64 + - name: owlMicro + type: object + default: + enabled: true + configs: + - windows_x64 + - linux_x64 + - name: tigerNoR2R + type: object + default: + enabled: true + configs: + - windows_x64 + - linux_x64 + - name: tigerNoPGO + type: object + default: + enabled: true + configs: + - windows_x64 + - linux_x64 + - name: tigerMicro + type: object + default: + enabled: true + configs: + - windows_x64 + - linux_x64 + - windown_x86 + - linux_musl_x64 + - name: tigerMonoAOT + type: object + default: + enabled: true + configs: + - linux_x64 + - name: tigerMonoInterpreter + type: object + default: + enabled: true + configs: + - linux_x64 + - name: tigerMono + type: object + default: + enabled: true + configs: + - linux_x64 + - name: androidMono + type: object + default: + enabled: true + configs: + - android_arm64 jobs: - template: /eng/pipelines/coreclr/templates/perf-build-jobs.yml@${{ parameters.runtimeRepoAlias }} From f1b3f254147185adab06c2fcb1123191869f4a56 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 17:12:37 +1000 Subject: [PATCH 35/45] Fix template conditions --- eng/performance/templates/run-performance-job.yml | 4 ++-- eng/performance/templates/runtime-perf-slow-jobs.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index e7b68d644a4..6c229440815 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -135,7 +135,7 @@ jobs: - ${{ each variable in parameters.variables }}: - ${{ variable }} steps: - - ${{ if parameters.isRuntimeJob }}: + - ${{ if eq(parameters.isRuntimeJob, true) }}: - checkout: ${{ parameters.runtimeRepoAlias }} path: s clean: true @@ -200,7 +200,7 @@ jobs: - ${{ elseif ne(parameters.framework, '') }}: - '--framework ${{ parameters.framework }}' - '--performance-repo-dir $(Build.SourcesDirectory)/performance' - - ${{ if parameters.isRuntimeJob }}: + - ${{ if eq(parameters.isRuntimeJob, true) }}: - '--runtime-repo-dir $(Build.SourcesDirectory)' - ${{ if ne(parameters.codeGenType, '') }}: - '--codegen-type ${{ parameters.codeGenType }}' diff --git a/eng/performance/templates/runtime-perf-slow-jobs.yml b/eng/performance/templates/runtime-perf-slow-jobs.yml index 9ccee741692..0ac0867fef4 100644 --- a/eng/performance/templates/runtime-perf-slow-jobs.yml +++ b/eng/performance/templates/runtime-perf-slow-jobs.yml @@ -6,7 +6,7 @@ parameters: jobParameters: {} jobs: - - ${{ if parameters.runScheduledJobs }}: + - ${{ if eq*parameters.runScheduledJobs, true) }}: - template: /eng/pipelines/coreclr/templates/perf-arm64-build-jobs.yml@${{ parameters.runtimeRepoAlias }} parameters: mono: true @@ -50,7 +50,7 @@ jobs: ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - - ${{ if parameters.runPrivateJobs }}: + - ${{ if eq(parameters.runPrivateJobs, true) }}: - template: /eng/pipelines/coreclr/templates/perf-arm64-build-jobs.yml@${{ parameters.runtimeRepoAlias }} parameters: coreclr: true From 49bff1984cd9b5fdf9407ef4221538e0334be4f3 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 17:45:08 +1000 Subject: [PATCH 36/45] Fix template condition --- eng/performance/templates/runtime-perf-slow-jobs.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/performance/templates/runtime-perf-slow-jobs.yml b/eng/performance/templates/runtime-perf-slow-jobs.yml index 0ac0867fef4..6ea0b0bc940 100644 --- a/eng/performance/templates/runtime-perf-slow-jobs.yml +++ b/eng/performance/templates/runtime-perf-slow-jobs.yml @@ -6,7 +6,7 @@ parameters: jobParameters: {} jobs: - - ${{ if eq*parameters.runScheduledJobs, true) }}: + - ${{ if eq(parameters.runScheduledJobs, true) }}: - template: /eng/pipelines/coreclr/templates/perf-arm64-build-jobs.yml@${{ parameters.runtimeRepoAlias }} parameters: mono: true From 67209e1c609fe2c9e25a308f0aac2fb5643baa9f Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 17:58:49 +1000 Subject: [PATCH 37/45] Set correct path to performance repository --- eng/performance/templates/run-performance-job.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/eng/performance/templates/run-performance-job.yml b/eng/performance/templates/run-performance-job.yml index 6c229440815..72f7ef51132 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/performance/templates/run-performance-job.yml @@ -121,11 +121,13 @@ jobs: value: 'Checked' - name: _BuildConfig value: $(buildConfigUpper) + - name: performanceRepoDir + value: $(Pipeline.Workspace)/s/performance - name: _projectFile ${{ if ne(parameters.projectFile, '') }}: value: ${{ parameters.projectFile }} ${{ else }}: - value: $(Build.SourcesDirectory)/performance/eng/performance/${{ parameters.projectFileName }} + value: $(performanceRepoDir)/eng/performance/${{ parameters.projectFileName }} - ${{ if eq(parameters.osGroup, 'windows') }}: - name: Python value: 'py -3' @@ -157,7 +159,7 @@ jobs: az storage blob download --auth-mode login --account-name pvscmdupload --container-name assets --name paint.net.5.0.3.portable.${{ parameters.archType }}.zip --file $(_pdnPath) - template: /eng/performance/templates/run-performance-job-script-step.yml parameters: - performanceRepoDir: $(Build.SourcesDirectory)/performance + performanceRepoDir: $(performanceRepoDir) runPerformanceJobArgs: - '--run-kind ${{ parameters.runKind }}' - '--architecture ${{ parameters.archType }}' @@ -199,7 +201,7 @@ jobs: - '--framework $(_Framework)' - ${{ elseif ne(parameters.framework, '') }}: - '--framework ${{ parameters.framework }}' - - '--performance-repo-dir $(Build.SourcesDirectory)/performance' + - '--performance-repo-dir $(performanceRepoDir)' - ${{ if eq(parameters.isRuntimeJob, true) }}: - '--runtime-repo-dir $(Build.SourcesDirectory)' - ${{ if ne(parameters.codeGenType, '') }}: From 6be2b25b72ad760deb454ef0a16733f819b4088b Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 19:18:33 +1000 Subject: [PATCH 38/45] Move pipelines to separate directory --- azure-pipelines.yml | 737 +----------------- .../runtime-ios-scenarios-perf-jobs.yml | 12 +- .../runtime-perf-jobs.yml | 28 +- .../runtime-perf-slow-jobs.yml | 93 +-- .../runtime-wasm-perf-jobs.yml | 10 +- eng/pipelines/sdk-perf-jobs.yml | 695 +++++++++++++++++ .../templates/build-machine-matrix.yml | 0 .../templates/download-artifact-step.yml | 0 .../download-specific-artifact-step.yml | 0 .../run-performance-job-script-step.yml | 0 .../templates/run-performance-job.yml | 4 +- .../templates/run-scenarios-job.yml | 2 +- .../templates/runtime-perf-job.yml | 18 +- .../templates/send-to-helix-step.yml | 0 14 files changed, 820 insertions(+), 779 deletions(-) rename eng/{performance/templates => pipelines}/runtime-ios-scenarios-perf-jobs.yml (89%) rename eng/{performance/templates => pipelines}/runtime-perf-jobs.yml (87%) rename eng/{performance/templates => pipelines}/runtime-perf-slow-jobs.yml (74%) rename eng/{performance/templates => pipelines}/runtime-wasm-perf-jobs.yml (89%) create mode 100644 eng/pipelines/sdk-perf-jobs.yml rename eng/{performance => pipelines}/templates/build-machine-matrix.yml (100%) rename eng/{performance => pipelines}/templates/download-artifact-step.yml (100%) rename eng/{performance => pipelines}/templates/download-specific-artifact-step.yml (100%) rename eng/{performance => pipelines}/templates/run-performance-job-script-step.yml (100%) rename eng/{performance => pipelines}/templates/run-performance-job.yml (99%) rename eng/{performance => pipelines}/templates/run-scenarios-job.yml (88%) rename eng/{performance => pipelines}/templates/runtime-perf-job.yml (95%) rename eng/{performance => pipelines}/templates/send-to-helix-step.yml (100%) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index eeaa1fbe035..4f8a5c07d2d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,6 +8,9 @@ parameters: - name: runScheduledPrivateJobs type: boolean default: false +- name: onlySanityCheck + type: boolean + default: false resources: containers: @@ -52,699 +55,41 @@ schedules: always: true jobs: - -###################################################### -# Public correctness jobs -###################################################### - -- ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: - - # Scenario benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: scenarios - projectFileName: scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 8.0 - - ## MAUI scenario benchmarks - #- template: /eng/performance/templates/build-machine-matrix.yml - # parameters: - # jobTemplate: /eng/performance/templates/run-scenarios-job.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # isPublic: true - # jobParameters: - # runKind: maui_scenarios - # projectFileName: maui_scenarios.proj - # channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - # - main - # - 8.0 - - # Blazor scenario benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: blazor_scenarios - projectFileName: blazor_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 8.0 - - # SDK scenario benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - - win-x86 - #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available - isPublic: true - jobParameters: - runKind: sdk_scenarios - projectFileName: sdk_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 8.0 - - # micro benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - win-x86 - isPublic: true - jobParameters: - runKind: micro - targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - main - - 8.0 - - # Ubuntux64 Default and NativeAOT micro benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: micro - targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - main - - nativeaot9.0 - - nativeaot8.0 - - 8.0 - - # net462 micro benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-rs5-x64 - isPublic: true - jobParameters: - runKind: micro_net462 - targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj - runCategories: 'runtime libraries' - channels: - - net462 - - # ML.NET benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: mlnet - targetCsproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj - runCategories: 'mldotnet' - channels: - - main - - 8.0 - - # F# benchmarks - - ${{ if false }}: # skipping, no useful benchmarks there currently - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: fsharp - targetCsproj: src\benchmarks\real-world\FSharp\FSharp.fsproj - runCategories: 'fsharp' - channels: - - main - - 8.0 - - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: fsharpmicro - targetCsproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj - runCategories: 'FSharpMicro' - channels: - - main - - 8.0 - - # bepuphysics benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: bepuphysics - targetCsproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj - runCategories: 'BepuPhysics' - channels: - - main - - 8.0 - - # ImageSharp benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: imagesharp - targetCsproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj - runCategories: 'ImageSharp' - channels: - - main - - 8.0 - - # Akade.IndexedSet benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: akadeindexedset - targetCsproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj - runCategories: 'AkadeIndexedSet' - channels: - - main - - 8.0 - - # Roslyn benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: roslyn - targetCsproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - runCategories: 'roslyn' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - 8.0 - - # ILLink benchmarks - # disabled because of: https://github.com/dotnet/performance/issues/3569 - # - template: /eng/performance/templates/build-machine-matrix.yml - # parameters: - # jobTemplate: /eng/performance/templates/run-performance-job.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # isPublic: true - # jobParameters: - # runKind: illink - # targetCsproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj - # runCategories: 'illink' - # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - # - main - - # NativeAOT scenario benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: nativeaot_scenarios - projectFileName: nativeaot_scenarios.proj - channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks - - main - - 8.0 - - # Powershell benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - isPublic: true - jobParameters: - runKind: powershell - targetCsproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj - runCategories: 'Public' - channels: - - main - - 8.0 - -########################################### -# Private Jobs -########################################### - -- ${{ if or(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: - - # Scenario benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: scenarios - projectFileName: scenarios.proj - channels: - - main - - 8.0 - - # Affinitized Scenario benchmarks (Initially just PDN) - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - - win-arm64 - - win-arm64-ampere - isPublic: false - jobParameters: - runKind: scenarios - projectFileName: scenarios_affinitized.proj - channels: - - main - - 8.0 - additionalJobIdentifier: 'Affinity_85' - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - - # Maui Android scenario benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-scenarios-job.yml - buildMachines: - - win-x64-android-arm64-pixel - - win-x64-android-arm64-galaxy - isPublic: false - jobParameters: - runKind: maui_scenarios_android - projectFileName: maui_scenarios_android.proj - dotnetVersionsLinks: - 9.0: ./eng/Version.Details.xml - channels: - - 8.0 - - # Maui iOS Mono scenario benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - dotnetVersionsLinks: - 9.0: ./eng/Version.Details.xml - channels: - - 8.0 - runtimeFlavor: mono - additionalJobIdentifier: Mono - - # Maui iOS Native AOT scenario benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-scenarios-job.yml - buildMachines: - - osx-x64-ios-arm64 - isPublic: false - jobParameters: - runKind: maui_scenarios_ios - projectFileName: maui_scenarios_ios.proj - dotnetVersionsLinks: - 9.0: ./eng/Version.Details.xml - channels: - - 8.0 - runtimeFlavor: coreclr - additionalJobIdentifier: CoreCLR - - ## Maui scenario benchmarks - #- template: /eng/performance/templates/build-machine-matrix.yml - # parameters: - # jobTemplate: /eng/performance/templates/run-scenarios-job.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # - win-arm64 - # - ubuntu-arm64-ampere - # isPublic: false - # jobParameters: - # runKind: maui_scenarios - # projectFileName: maui_scenarios.proj - # channels: - # - main - # - 8.0 - - # NativeAOT scenario benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - isPublic: false - jobParameters: - runKind: nativeaot_scenarios - projectFileName: nativeaot_scenarios.proj - channels: - - main - - 8.0 - -################################################ -# Scheduled Private jobs -################################################ - -# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -- ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: - - # SDK scenario benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - - win-x86 - #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available - isPublic: false - jobParameters: - runKind: sdk_scenarios - projectFileName: sdk_scenarios.proj - channels: - - main - - 8.0 - - # Blazor 3.2 scenario benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - isPublic: false - jobParameters: - runKind: blazor_scenarios - projectFileName: blazor_scenarios.proj - channels: - - main - - 8.0 - - # F# benchmarks - - ${{ if false }}: # skipping, no useful benchmarks there currently - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: fsharp - targetCsproj: src\benchmarks\real-world\FSharp\FSharp.fsproj - runCategories: 'fsharp' - channels: - - main - - 8.0 - - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: fsharpmicro - targetCsproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj - runCategories: 'FSharpMicro' - channels: - - main - - 8.0 - - # bepuphysics benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: bepuphysics - targetCsproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj - runCategories: 'BepuPhysics' - channels: - - main - - 8.0 - - # ImageSharp benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: imagesharp - targetCsproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj - runCategories: 'ImageSharp' - channels: - - main - - 8.0 - - # Akade.IndexedSet benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: akadeindexedset - targetCsproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj - runCategories: 'AkadeIndexedSet' - channels: - - main - - # ML.NET benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: mlnet - targetCsproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj - runCategories: 'mldotnet' - channels: - - main - - 8.0 - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - - # Roslyn benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: roslyn - targetCsproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - runCategories: 'roslyn' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - 8.0 - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - - # ILLink benchmarks - # disabled because of: https://github.com/dotnet/performance/issues/3569 - # - template: /eng/performance/templates/build-machine-matrix.yml - # parameters: - # jobTemplate: /eng/performance/templates/run-performance-job.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) - # isPublic: false - # jobParameters: - # runKind: illink - # targetCsproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj - # runCategories: 'illink' - # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - # - main - # - 8.0 - - # Powershell benchmarks - - template: /eng/performance/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/performance/templates/run-performance-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64 - isPublic: false - jobParameters: - runKind: powershell - targetCsproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj - runCategories: 'Public Internal' - channels: - - main - - 8.0 - - # Secret Sync - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 - inputs: - version: 6.x - - - task: DeleteFiles@1 - inputs: - Contents: global.json - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} - -################################################ -# Manually Triggered Job -################################################ - - -- ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Manual'), not(parameters.runPublicJobs), not(parameters.runScheduledPrivateJobs), not(parameters.runPrivateJobs)) }}: - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 - inputs: - version: 6.x - - - task: DeleteFiles@1 - inputs: - Contents: global.json - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} - -################################################ -# Scheduled Private jobs (Weekly) -################################################ - -# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job -- ${{ if and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), contains(variables['Build.QueuedBy'], 'Weekly')) }}: - - job: Synchronize - pool: - name: NetCore1ESPool-Internal-NoMSI - demands: ImageOverride -equals 1es-windows-2019 - steps: - - task: UseDotNet@2 - displayName: Install .NET 6.0 - inputs: - version: 6.x - - - task: DeleteFiles@1 - inputs: - Contents: global.json - - - script: dotnet tool restore - - - task: AzureCLI@2 - inputs: - azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} + - template: /eng/pipelines/sdk-perf-jobs.yml + parameters: + ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: + runPublicJobs: ${{ parameters.runPublicJobs }} + ${{ if ne(variables['System.TeamProject'], 'public') }}: + ${{ if or(and(notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: + runPrivateJobs: ${{ parameters.runPrivateJobs }} + ${{ if or(eq(variables['Build.CronSchedule.DisplayName'], 'Every 12 hours build'), parameters.runScheduledPrivateJobs) }}: + runScheduledPrivateJobs: ${{ parameters.runScheduledPrivateJobs }} + jobParameters: + ${{ if parameters.onlySanityCheck }}: + onlySanityCheck: true + + - ${{ if and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule', 'Manual')) }}: + # Secret Sync + - job: Synchronize + pool: + name: NetCore1ESPool-Internal-NoMSI + demands: ImageOverride -equals 1es-windows-2019 + steps: + - task: UseDotNet@2 + displayName: Install .NET 6.0 + inputs: + version: 6.x + + - task: DeleteFiles@1 + inputs: + Contents: global.json + + - script: dotnet tool restore + + - task: AzureCLI@2 + inputs: + azureSubscription: .NET Performance (790c4451-dad9-4fda-af8b-10bd9ca328fa) + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + Get-ChildItem .vault-config/*.yaml |% { dotnet secret-manager synchronize $_} \ No newline at end of file diff --git a/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml b/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml similarity index 89% rename from eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml rename to eng/pipelines/runtime-ios-scenarios-perf-jobs.yml index 858ddfb8e35..fff20ce6dbd 100644 --- a/eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml +++ b/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml @@ -12,7 +12,7 @@ jobs: # run mono iOS scenarios scenarios HybridGlobalization - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -33,7 +33,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -55,7 +55,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -77,7 +77,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -100,7 +100,7 @@ jobs: # run NativeAOT iOS scenarios HybridGlobalization - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -120,7 +120,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: diff --git a/eng/performance/templates/runtime-perf-jobs.yml b/eng/pipelines/runtime-perf-jobs.yml similarity index 87% rename from eng/performance/templates/runtime-perf-jobs.yml rename to eng/pipelines/runtime-perf-jobs.yml index 34d2365f08d..e3bd7f553b9 100644 --- a/eng/performance/templates/runtime-perf-jobs.yml +++ b/eng/pipelines/runtime-perf-jobs.yml @@ -93,7 +93,7 @@ jobs: perfBranch: ${{ parameters.perfBranch }} # Build and run iOS Mono and NativeAOT scenarios - - template: /eng/performance/templates/runtime-ios-scenarios-perf-jobs.yml + - template: /eng/pipelines/runtime-ios-scenarios-perf-jobs.yml parameters: hybridGlobalization: True runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} @@ -103,7 +103,7 @@ jobs: # run android scenarios - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -122,7 +122,7 @@ jobs: # run mono microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -140,7 +140,7 @@ jobs: # run mono interpreter perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -159,7 +159,7 @@ jobs: # run mono aot microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? buildConfig: release runtimeFlavor: aot platforms: @@ -178,7 +178,7 @@ jobs: # run coreclr perftiger microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -198,7 +198,7 @@ jobs: # run coreclr perftiger microbenchmarks no dynamic pgo perf jobs - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -216,7 +216,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -235,7 +235,7 @@ jobs: # run coreclr perftiger microbenchmarks no R2R perf jobs - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -253,7 +253,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -272,7 +272,7 @@ jobs: # run coreclr perfowl microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -290,7 +290,7 @@ jobs: # run coreclr perfviper microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -308,7 +308,7 @@ jobs: # run coreclr perfviper microbenchmarks perf jitoptrepeat jobs - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -328,7 +328,7 @@ jobs: # run coreclr crossgen perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: diff --git a/eng/performance/templates/runtime-perf-slow-jobs.yml b/eng/pipelines/runtime-perf-slow-jobs.yml similarity index 74% rename from eng/performance/templates/runtime-perf-slow-jobs.yml rename to eng/pipelines/runtime-perf-slow-jobs.yml index 6ea0b0bc940..49f78fe8671 100644 --- a/eng/performance/templates/runtime-perf-slow-jobs.yml +++ b/eng/pipelines/runtime-perf-slow-jobs.yml @@ -14,7 +14,7 @@ jobs: # run arm64 interpreter jobs for mono - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -34,7 +34,7 @@ jobs: # run arm64 jit jobs for mono - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: mono platforms: @@ -59,7 +59,7 @@ jobs: # run mono aot microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? buildConfig: release runtimeFlavor: aot platforms: @@ -79,7 +79,7 @@ jobs: # run coreclr Linux arm64 ampere microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -97,7 +97,7 @@ jobs: #run coreclr Linux arm64 ampere no dynamic pgo microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -117,7 +117,7 @@ jobs: #run coreclr Linux arm64 ampere no R2R microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -137,7 +137,7 @@ jobs: # run coreclr Windows arm64 microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -154,7 +154,7 @@ jobs: # run coreclr Windows arm64 ampere microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -172,7 +172,7 @@ jobs: # run coreclr Windows arm64 ampere no dynamic pgo microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -192,7 +192,7 @@ jobs: # run coreclr Windows arm64 ampere no R2R microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: coreclr platforms: @@ -210,40 +210,41 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} # Uncomment once we fix https://github.com/dotnet/performance/issues/1950 - # # run coreclr linux crossgen perf job - # - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} - # parameters: - # jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} - # buildConfig: release - # runtimeFlavor: coreclr - # platforms: - # - linux_arm64 - # jobParameters: - # liveLibrariesBuildConfig: Release - # projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj - # runKind: crossgen_scenarios - # isScenario: true - # logicalMachine: 'perfa64' - # runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} - # performanceRepoAlias: ${{ parameters.performanceRepoAlias }} - # ${{ each parameter in parameters.jobParameters }}: - # ${{ parameter.key }}: ${{ parameter.value }} + - ${{ if false }}: + # run coreclr linux crossgen perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - linux_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj + runKind: crossgen_scenarios + isScenario: true + logicalMachine: 'perfa64' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} - # # run coreclr windows crossgen perf job - # - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} - # parameters: - # jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} - # buildConfig: release - # runtimeFlavor: coreclr - # platforms: - # - windows_arm64 - # jobParameters: - # liveLibrariesBuildConfig: Release - # projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj - # runKind: crossgen_scenarios - # isScenario: true - # logicalMachine: 'perfsurf' - # runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} - # performanceRepoAlias: ${{ parameters.performanceRepoAlias }} - # ${{ each parameter in parameters.jobParameters }}: - # ${{ parameter.key }}: ${{ parameter.value }} + # run coreclr windows crossgen perf job + - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} + parameters: + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_arm64 + jobParameters: + liveLibrariesBuildConfig: Release + projectFile: $(Build.SourcesDirectory)/eng/testing/performance/crossgen_perf.proj + runKind: crossgen_scenarios + isScenario: true + logicalMachine: 'perfsurf' + runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} + performanceRepoAlias: ${{ parameters.performanceRepoAlias }} + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/performance/templates/runtime-wasm-perf-jobs.yml b/eng/pipelines/runtime-wasm-perf-jobs.yml similarity index 89% rename from eng/performance/templates/runtime-wasm-perf-jobs.yml rename to eng/pipelines/runtime-wasm-perf-jobs.yml index 49d2df6450f..8eca2b36a8f 100644 --- a/eng/performance/templates/runtime-wasm-perf-jobs.yml +++ b/eng/pipelines/runtime-wasm-perf-jobs.yml @@ -15,7 +15,7 @@ jobs: #run mono wasm microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? buildConfig: Release runtimeFlavor: wasm platforms: @@ -37,7 +37,7 @@ jobs: #run mono wasm aot microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # note: should we move this file out of coreclr tempelates because it contains mono jobs? + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # note: should we move this file out of coreclr tempelates because it contains mono jobs? buildconfig: Release runtimeflavor: wasm platforms: @@ -60,7 +60,7 @@ jobs: # run mono wasm interpreter (default) microbenchmarks perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: wasm platforms: @@ -86,7 +86,7 @@ jobs: - ${{if not(in(variables['Build.DefinitionName'], 'runtime-wasm-perf')) }}: - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # note: should we move this file out of coreclr tempelates because it contains mono jobs? + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} # note: should we move this file out of coreclr tempelates because it contains mono jobs? buildconfig: release runtimeflavor: wasm platforms: @@ -110,7 +110,7 @@ jobs: # run mono wasm blazor perf job - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: - jobTemplate: /eng/performance/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} + jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} buildConfig: release runtimeFlavor: wasm platforms: diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml new file mode 100644 index 00000000000..23f39fbe8ef --- /dev/null +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -0,0 +1,695 @@ +parameters: + - name: runPublicJobs + type: boolean + default: false + - name: runPrivateJobs + type: boolean + default: false + - name: runScheduledPrivateJobs + type: boolean + default: false + - name: jobParameters + type: object + default: {} + +jobs: +###################################################### +# Public correctness jobs +###################################################### + +- ${{ if parameters.runPublicJobs }}: + + # Scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: scenarios + projectFileName: scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # MAUI scenario benchmarks + - ${{ if false }}: + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: maui_scenarios + projectFileName: maui_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Blazor scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: blazor_scenarios + projectFileName: blazor_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # SDK scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - win-x86 + #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available + isPublic: true + jobParameters: + runKind: sdk_scenarios + projectFileName: sdk_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # micro benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - win-x86 + isPublic: true + jobParameters: + runKind: micro + targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - main + - 8.0 + + # Ubuntux64 Default and NativeAOT micro benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: micro + targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - main + - nativeaot9.0 + - nativeaot8.0 + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # net462 micro benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-rs5-x64 + isPublic: true + jobParameters: + runKind: micro_net462 + targetCsproj: src\benchmarks\micro\MicroBenchmarks.csproj + runCategories: 'runtime libraries' + channels: + - net462 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # ML.NET benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: mlnet + targetCsproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + runCategories: 'mldotnet' + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # F# benchmarks + - ${{ if false }}: # skipping, no useful benchmarks there currently + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: fsharp + targetCsproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + runCategories: 'fsharp' + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: fsharpmicro + targetCsproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj + runCategories: 'FSharpMicro' + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # bepuphysics benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: bepuphysics + targetCsproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + runCategories: 'BepuPhysics' + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # ImageSharp benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: imagesharp + targetCsproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + runCategories: 'ImageSharp' + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Akade.IndexedSet benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: akadeindexedset + targetCsproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj + runCategories: 'AkadeIndexedSet' + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Roslyn benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: roslyn + targetCsproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runCategories: 'roslyn' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # ILLink benchmarks + # disabled because of: https://github.com/dotnet/performance/issues/3569 + - ${{ if false }}: + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: illink + targetCsproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + runCategories: 'illink' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # NativeAOT scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: nativeaot_scenarios + projectFileName: nativeaot_scenarios.proj + channels: # for public jobs we want to make sure that the PRs don't break any of the supported frameworks + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Powershell benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + isPublic: true + jobParameters: + runKind: powershell + targetCsproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj + runCategories: 'Public' + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +########################################### +# Private Jobs +########################################### + +- ${{ if parameters.runPrivateJobs }}: + + # Scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: scenarios + projectFileName: scenarios.proj + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Affinitized Scenario benchmarks (Initially just PDN) + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - win-arm64 + - win-arm64-ampere + isPublic: false + jobParameters: + runKind: scenarios + projectFileName: scenarios_affinitized.proj + channels: + - main + - 8.0 + additionalJobIdentifier: 'Affinity_85' + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui Android scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64-android-arm64-pixel + - win-x64-android-arm64-galaxy + isPublic: false + jobParameters: + runKind: maui_scenarios_android + projectFileName: maui_scenarios_android.proj + dotnetVersionsLinks: + 9.0: ./eng/Version.Details.xml + channels: + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS Mono scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + dotnetVersionsLinks: + 9.0: ./eng/Version.Details.xml + channels: + - 8.0 + runtimeFlavor: mono + additionalJobIdentifier: Mono + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui iOS Native AOT scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - osx-x64-ios-arm64 + isPublic: false + jobParameters: + runKind: maui_scenarios_ios + projectFileName: maui_scenarios_ios.proj + dotnetVersionsLinks: + 9.0: ./eng/Version.Details.xml + channels: + - 8.0 + runtimeFlavor: coreclr + additionalJobIdentifier: CoreCLR + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Maui scenario benchmarks + - ${{ if false }}: + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: maui_scenarios + projectFileName: maui_scenarios.proj + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # NativeAOT scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + isPublic: false + jobParameters: + runKind: nativeaot_scenarios + projectFileName: nativeaot_scenarios.proj + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + +################################################ +# Scheduled Private jobs +################################################ + +# Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job +- ${{ if parameters.runScheduledPrivateJobs }}: + + # SDK scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - win-x86 + #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available + isPublic: false + jobParameters: + runKind: sdk_scenarios + projectFileName: sdk_scenarios.proj + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Blazor 3.2 scenario benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + isPublic: false + jobParameters: + runKind: blazor_scenarios + projectFileName: blazor_scenarios.proj + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # F# benchmarks + - ${{ if false }}: # skipping, no useful benchmarks there currently + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: fsharp + targetCsproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + runCategories: 'fsharp' + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: fsharpmicro + targetCsproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj + runCategories: 'FSharpMicro' + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # bepuphysics benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: bepuphysics + targetCsproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + runCategories: 'BepuPhysics' + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # ImageSharp benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: imagesharp + targetCsproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + runCategories: 'ImageSharp' + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Akade.IndexedSet benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: akadeindexedset + targetCsproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj + runCategories: 'AkadeIndexedSet' + channels: + - main + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # ML.NET benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: mlnet + targetCsproj: src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj + runCategories: 'mldotnet' + channels: + - main + - 8.0 + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # Roslyn benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: roslyn + targetCsproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runCategories: 'roslyn' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + - 8.0 + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} + + # ILLink benchmarks + # disabled because of: https://github.com/dotnet/performance/issues/3569 + - ${{ if false }}: + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) + isPublic: false + jobParameters: + runKind: illink + targetCsproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + runCategories: 'illink' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + - 8.0 + + # Powershell benchmarks + - template: /eng/pipelines/templates/build-machine-matrix.yml + parameters: + jobTemplate: /eng/pipelines/templates/run-performance-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64 + isPublic: false + jobParameters: + runKind: powershell + targetCsproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj + runCategories: 'Public Internal' + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} \ No newline at end of file diff --git a/eng/performance/templates/build-machine-matrix.yml b/eng/pipelines/templates/build-machine-matrix.yml similarity index 100% rename from eng/performance/templates/build-machine-matrix.yml rename to eng/pipelines/templates/build-machine-matrix.yml diff --git a/eng/performance/templates/download-artifact-step.yml b/eng/pipelines/templates/download-artifact-step.yml similarity index 100% rename from eng/performance/templates/download-artifact-step.yml rename to eng/pipelines/templates/download-artifact-step.yml diff --git a/eng/performance/templates/download-specific-artifact-step.yml b/eng/pipelines/templates/download-specific-artifact-step.yml similarity index 100% rename from eng/performance/templates/download-specific-artifact-step.yml rename to eng/pipelines/templates/download-specific-artifact-step.yml diff --git a/eng/performance/templates/run-performance-job-script-step.yml b/eng/pipelines/templates/run-performance-job-script-step.yml similarity index 100% rename from eng/performance/templates/run-performance-job-script-step.yml rename to eng/pipelines/templates/run-performance-job-script-step.yml diff --git a/eng/performance/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml similarity index 99% rename from eng/performance/templates/run-performance-job.yml rename to eng/pipelines/templates/run-performance-job.yml index 72f7ef51132..47783967f71 100644 --- a/eng/performance/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -157,7 +157,7 @@ jobs: scriptLocation: 'inlineScript' inlineScript: | az storage blob download --auth-mode login --account-name pvscmdupload --container-name assets --name paint.net.5.0.3.portable.${{ parameters.archType }}.zip --file $(_pdnPath) - - template: /eng/performance/templates/run-performance-job-script-step.yml + - template: /eng/pipelines/templates/run-performance-job-script-step.yml parameters: performanceRepoDir: $(performanceRepoDir) runPerformanceJobArgs: @@ -232,7 +232,7 @@ jobs: - '--pdn-path "$(_pdnPath)"' - ${{ if ne(parameters.additionalSetupParameters, '') }}: - '${{ parameters.additionalSetupParameters }}' - - template: /eng/performance/templates/send-to-helix-step.yml + - template: /eng/pipelines/templates/send-to-helix-step.yml parameters: osGroup: ${{ parameters.osGroup }} projectFile: $(_projectFile) \ No newline at end of file diff --git a/eng/performance/templates/run-scenarios-job.yml b/eng/pipelines/templates/run-scenarios-job.yml similarity index 88% rename from eng/performance/templates/run-scenarios-job.yml rename to eng/pipelines/templates/run-scenarios-job.yml index 3ccaf4f67ec..2c606ce9418 100644 --- a/eng/performance/templates/run-scenarios-job.yml +++ b/eng/pipelines/templates/run-scenarios-job.yml @@ -4,7 +4,7 @@ parameters: # Please see run-performance-job.yml for additional parameters that can be specified as they will be forwarded through jobs: -- template: /eng/performance/templates/run-performance-job.yml +- template: /eng/pipelines/templates/run-performance-job.yml parameters: isScenario: true downloadPdn: ${{ and(eq(parameters.osGroup, 'windows'), ne(parameters.archType, 'x86')) }} diff --git a/eng/performance/templates/runtime-perf-job.yml b/eng/pipelines/templates/runtime-perf-job.yml similarity index 95% rename from eng/performance/templates/runtime-perf-job.yml rename to eng/pipelines/templates/runtime-perf-job.yml index 1249cfff30f..3fefda72371 100644 --- a/eng/performance/templates/runtime-perf-job.yml +++ b/eng/pipelines/templates/runtime-perf-job.yml @@ -22,7 +22,7 @@ parameters: # Please see run-performance-job.yml for additional parameters that can be specified as they will be forwarded through jobs: -- template: /eng/performance/templates/run-performance-job.yml +- template: /eng/pipelines/templates/run-performance-job.yml parameters: ${{ each parameter in parameters }}: ${{ if notIn(parameter.key, 'steps', 'variables', 'framework') }}: @@ -102,14 +102,14 @@ jobs: - ${{ if eq(parameters.runtimeType, 'wasm') }}: # Download wasm - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: - - template: /eng/performance/templates/download-artifact-step.yml + - template: /eng/pipelines/templates/runtime-perf-job.yml parameters: unpackFolder: $(librariesDownloadDir)/BrowserWasm artifactFileName: BrowserWasm.tar.gz artifactName: BrowserWasm displayName: BrowserWasm - ${{ if ne(parameters.downloadSpecificBuild.buildId, '') }}: - - template: /eng/performance/templates/download-specific-artifact-step.yml + - template: /eng/pipelines/templates/download-specific-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/BrowserWasm artifactFileName: BrowserWasm.tar.gz @@ -128,14 +128,14 @@ jobs: displayName: "Create wasm directory (Linux)" - ${{ elseif eq(parameters.codeGenType, 'AOT') }}: # Download mono AOT - - template: /eng/performance/templates/download-artifact-step.yml + - template: /eng/pipelines/templates/runtime-perf-job.yml parameters: unpackFolder: $(librariesDownloadDir)/bin artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_coreclr$(archiveExtension)' artifactName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_coreclr' displayName: 'Runtime artifacts' - - template: /eng/performance/templates/download-artifact-step.yml + - template: /eng/pipelines/templates/runtime-perf-job.yml parameters: unpackFolder: $(librariesDownloadDir)/LinuxMonoAOT artifactFileName: LinuxMonoAOT${{ parameters.archType }}.tar.gz @@ -145,7 +145,7 @@ jobs: - script: "mkdir -p $(librariesDownloadDir)/bin/aot;mkdir -p $(librariesDownloadDir)/bin/aot/pack;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/mono/linux.${{ parameters.archType }}.Release/cross/linux-${{ parameters.archType }}/* $(librariesDownloadDir)/bin/aot;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/microsoft.netcore.app.runtime.linux-${{ parameters.archType }}/Release/* $(librariesDownloadDir)/bin/aot/pack" displayName: "Create aot directory (Linux)" - ${{ elseif in(parameters.runtimeType, 'coreclr', 'mono') }}: - - template: /eng/performance/templates/download-artifact-step.yml + - template: /eng/pipelines/templates/runtime-perf-job.yml parameters: unpackFolder: $(librariesDownloadDir)/bin artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_${{ parameters.runtimeType }}$(archiveExtension)' @@ -153,14 +153,14 @@ jobs: displayName: 'Runtime artifacts' - ${{ elseif eq(parameters.runtimeType, 'AndroidMono')}}: # Download artifacts for Android Testing - - template: /eng/performance/templates/download-artifact-step.yml + - template: /eng/pipelines/templates/runtime-perf-job.yml parameters: unpackFolder: $(builtAppDir)/androidHelloWorld cleanUnpackFolder: false artifactFileName: 'AndroidMonoarm64.tar.gz' artifactName: 'AndroidMonoarm64' displayName: 'Mono Android HelloWorld' - - template: /eng/performance/templates/download-artifact-step.yml + - template: /eng/pipelines/templates/runtime-perf-job.yml parameters: unpackFolder: $(builtAppDir) cleanUnpackFolder: false @@ -169,7 +169,7 @@ jobs: displayName: 'Mono Android BDN Apk' - ${{ elseif or(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.runtimeType, 'iOSNativeAOT')) }}: # Download iOSMono and Native AOT tests - - template: /eng/performance/templates/download-artifact-step.yml + - template: /eng/pipelines/templates/runtime-perf-job.yml parameters: unpackFolder: $(builtAppDir)/iosHelloWorld cleanUnpackFolder: false diff --git a/eng/performance/templates/send-to-helix-step.yml b/eng/pipelines/templates/send-to-helix-step.yml similarity index 100% rename from eng/performance/templates/send-to-helix-step.yml rename to eng/pipelines/templates/send-to-helix-step.yml From c2289fc17d9b87daa9e9f54ac8ed2fe17116994b Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 19:28:42 +1000 Subject: [PATCH 39/45] Fix incorrect template for download artifact step --- eng/pipelines/templates/runtime-perf-job.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/eng/pipelines/templates/runtime-perf-job.yml b/eng/pipelines/templates/runtime-perf-job.yml index 3fefda72371..e1d0dca3b3b 100644 --- a/eng/pipelines/templates/runtime-perf-job.yml +++ b/eng/pipelines/templates/runtime-perf-job.yml @@ -102,7 +102,7 @@ jobs: - ${{ if eq(parameters.runtimeType, 'wasm') }}: # Download wasm - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: - - template: /eng/pipelines/templates/runtime-perf-job.yml + - template: /eng/pipelines/templates/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/BrowserWasm artifactFileName: BrowserWasm.tar.gz @@ -128,14 +128,14 @@ jobs: displayName: "Create wasm directory (Linux)" - ${{ elseif eq(parameters.codeGenType, 'AOT') }}: # Download mono AOT - - template: /eng/pipelines/templates/runtime-perf-job.yml + - template: /eng/pipelines/templates/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/bin artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_coreclr$(archiveExtension)' artifactName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_coreclr' displayName: 'Runtime artifacts' - - template: /eng/pipelines/templates/runtime-perf-job.yml + - template: /eng/pipelines/templates/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/LinuxMonoAOT artifactFileName: LinuxMonoAOT${{ parameters.archType }}.tar.gz @@ -145,7 +145,7 @@ jobs: - script: "mkdir -p $(librariesDownloadDir)/bin/aot;mkdir -p $(librariesDownloadDir)/bin/aot/pack;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/mono/linux.${{ parameters.archType }}.Release/cross/linux-${{ parameters.archType }}/* $(librariesDownloadDir)/bin/aot;cp -r $(librariesDownloadDir)/LinuxMonoAOT/artifacts/bin/microsoft.netcore.app.runtime.linux-${{ parameters.archType }}/Release/* $(librariesDownloadDir)/bin/aot/pack" displayName: "Create aot directory (Linux)" - ${{ elseif in(parameters.runtimeType, 'coreclr', 'mono') }}: - - template: /eng/pipelines/templates/runtime-perf-job.yml + - template: /eng/pipelines/templates/download-artifact-step.yml parameters: unpackFolder: $(librariesDownloadDir)/bin artifactFileName: 'BuildArtifacts_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_$(buildConfigUpper)_${{ parameters.runtimeType }}$(archiveExtension)' @@ -153,14 +153,14 @@ jobs: displayName: 'Runtime artifacts' - ${{ elseif eq(parameters.runtimeType, 'AndroidMono')}}: # Download artifacts for Android Testing - - template: /eng/pipelines/templates/runtime-perf-job.yml + - template: /eng/pipelines/templates/download-artifact-step.yml parameters: unpackFolder: $(builtAppDir)/androidHelloWorld cleanUnpackFolder: false artifactFileName: 'AndroidMonoarm64.tar.gz' artifactName: 'AndroidMonoarm64' displayName: 'Mono Android HelloWorld' - - template: /eng/pipelines/templates/runtime-perf-job.yml + - template: /eng/pipelines/templates/download-artifact-step.yml parameters: unpackFolder: $(builtAppDir) cleanUnpackFolder: false @@ -169,7 +169,7 @@ jobs: displayName: 'Mono Android BDN Apk' - ${{ elseif or(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.runtimeType, 'iOSNativeAOT')) }}: # Download iOSMono and Native AOT tests - - template: /eng/pipelines/templates/runtime-perf-job.yml + - template: /eng/pipelines/templates/download-artifact-step.yml parameters: unpackFolder: $(builtAppDir)/iosHelloWorld cleanUnpackFolder: false From c9f69e1e7ffcb2da5b1d617db23ff30ba2cf3801 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 19:31:04 +1000 Subject: [PATCH 40/45] Fix unbalanced parentheses --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4f8a5c07d2d..6afa5e49abe 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -60,7 +60,7 @@ jobs: ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: runPublicJobs: ${{ parameters.runPublicJobs }} ${{ if ne(variables['System.TeamProject'], 'public') }}: - ${{ if or(and(notin(variables['Build.Reason'], 'PullRequest', 'Manual')), parameters.runPrivateJobs) }}: + ${{ if or(and(notin(variables['Build.Reason'], 'PullRequest', 'Manual'), parameters.runPrivateJobs) }}: runPrivateJobs: ${{ parameters.runPrivateJobs }} ${{ if or(eq(variables['Build.CronSchedule.DisplayName'], 'Every 12 hours build'), parameters.runScheduledPrivateJobs) }}: runScheduledPrivateJobs: ${{ parameters.runScheduledPrivateJobs }} From ecc7def46a125d9d074519ae8cd0854d50a4878b Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 19:33:14 +1000 Subject: [PATCH 41/45] Fix more unbalanced parentheses --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6afa5e49abe..c37e291adb7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -60,7 +60,7 @@ jobs: ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: runPublicJobs: ${{ parameters.runPublicJobs }} ${{ if ne(variables['System.TeamProject'], 'public') }}: - ${{ if or(and(notin(variables['Build.Reason'], 'PullRequest', 'Manual'), parameters.runPrivateJobs) }}: + ${{ if or(notin(variables['Build.Reason'], 'PullRequest', 'Manual'), parameters.runPrivateJobs) }}: runPrivateJobs: ${{ parameters.runPrivateJobs }} ${{ if or(eq(variables['Build.CronSchedule.DisplayName'], 'Every 12 hours build'), parameters.runScheduledPrivateJobs) }}: runScheduledPrivateJobs: ${{ parameters.runScheduledPrivateJobs }} From 7a191f82707e07be22114269e6d5ad6e98f8ba3b Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Wed, 15 Jan 2025 19:34:43 +1000 Subject: [PATCH 42/45] Fix indentation --- eng/pipelines/sdk-perf-jobs.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/eng/pipelines/sdk-perf-jobs.yml b/eng/pipelines/sdk-perf-jobs.yml index 23f39fbe8ef..3a121657e63 100644 --- a/eng/pipelines/sdk-perf-jobs.yml +++ b/eng/pipelines/sdk-perf-jobs.yml @@ -428,22 +428,22 @@ jobs: # Maui scenario benchmarks - ${{ if false }}: - template: /eng/pipelines/templates/build-machine-matrix.yml - parameters: - jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - runKind: maui_scenarios - projectFileName: maui_scenarios.proj - channels: - - main - - 8.0 - ${{ each parameter in parameters.jobParameters }}: - ${{ parameter.key }}: ${{ parameter.value }} + parameters: + jobTemplate: /eng/pipelines/templates/run-scenarios-job.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + runKind: maui_scenarios + projectFileName: maui_scenarios.proj + channels: + - main + - 8.0 + ${{ each parameter in parameters.jobParameters }}: + ${{ parameter.key }}: ${{ parameter.value }} # NativeAOT scenario benchmarks - template: /eng/pipelines/templates/build-machine-matrix.yml From bcf091a02763d6ce3f4bd6c9c5dbaf915c081f85 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Tue, 21 Jan 2025 19:41:21 +1000 Subject: [PATCH 43/45] Address PR comments --- eng/pipelines/runtime-ios-scenarios-perf-jobs.yml | 4 ++-- eng/pipelines/runtime-perf-jobs.yml | 2 +- ...ntime-perf-slow-jobs.yml => runtime-slow-perf-jobs.yml} | 4 ++-- eng/pipelines/runtime-wasm-perf-jobs.yml | 2 +- eng/pipelines/templates/run-performance-job.yml | 7 +++---- 5 files changed, 9 insertions(+), 10 deletions(-) rename eng/pipelines/{runtime-perf-slow-jobs.yml => runtime-slow-perf-jobs.yml} (97%) diff --git a/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml b/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml index fff20ce6dbd..472c27429fa 100644 --- a/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml +++ b/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml @@ -5,11 +5,11 @@ parameters: jobParameters: {} jobs: - - template: /eng/pipelines/coreclr/templates/perf-ios-scenarios-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/performance/templates/perf-ios-scenarios-build-jobs.yml@${{ parameters.runtimeRepoAlias }} parameters: hybridGlobalization: ${{ parameters.hybridGlobalization }} - # run mono iOS scenarios scenarios HybridGlobalization + # run mono iOS scenarios HybridGlobalization - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} diff --git a/eng/pipelines/runtime-perf-jobs.yml b/eng/pipelines/runtime-perf-jobs.yml index e3bd7f553b9..427d815b04f 100644 --- a/eng/pipelines/runtime-perf-jobs.yml +++ b/eng/pipelines/runtime-perf-jobs.yml @@ -88,7 +88,7 @@ parameters: - android_arm64 jobs: - - template: /eng/pipelines/coreclr/templates/perf-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/performance/templates/perf-build-jobs.yml@${{ parameters.runtimeRepoAlias }} parameters: perfBranch: ${{ parameters.perfBranch }} diff --git a/eng/pipelines/runtime-perf-slow-jobs.yml b/eng/pipelines/runtime-slow-perf-jobs.yml similarity index 97% rename from eng/pipelines/runtime-perf-slow-jobs.yml rename to eng/pipelines/runtime-slow-perf-jobs.yml index 49f78fe8671..c818bbca2ed 100644 --- a/eng/pipelines/runtime-perf-slow-jobs.yml +++ b/eng/pipelines/runtime-slow-perf-jobs.yml @@ -7,7 +7,7 @@ parameters: jobs: - ${{ if eq(parameters.runScheduledJobs, true) }}: - - template: /eng/pipelines/coreclr/templates/perf-arm64-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/performance/templates/perf-arm64-build-jobs.yml@${{ parameters.runtimeRepoAlias }} parameters: mono: true @@ -51,7 +51,7 @@ jobs: ${{ parameter.key }}: ${{ parameter.value }} - ${{ if eq(parameters.runPrivateJobs, true) }}: - - template: /eng/pipelines/coreclr/templates/perf-arm64-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/performance/templates/perf-arm64-build-jobs.yml@${{ parameters.runtimeRepoAlias }} parameters: coreclr: true monoAot: true diff --git a/eng/pipelines/runtime-wasm-perf-jobs.yml b/eng/pipelines/runtime-wasm-perf-jobs.yml index 8eca2b36a8f..a4b32607f52 100644 --- a/eng/pipelines/runtime-wasm-perf-jobs.yml +++ b/eng/pipelines/runtime-wasm-perf-jobs.yml @@ -9,7 +9,7 @@ parameters: jobs: - ${{ if eq(parameters.downloadSpecificBuild.buildId, '') }}: # build mono on wasm - if not using an existing build - - template: /eng/pipelines/coreclr/templates/perf-wasm-build-jobs.yml@${{ parameters.runtimeRepoAlias }} + - template: /eng/pipelines/performance/templates/perf-wasm-build-jobs.yml@${{ parameters.runtimeRepoAlias }} - ${{ if eq(parameters.runProfile, 'non-v8') }}: #run mono wasm microbenchmarks perf job diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index 47783967f71..e50a71ef251 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -128,11 +128,10 @@ jobs: value: ${{ parameters.projectFile }} ${{ else }}: value: $(performanceRepoDir)/eng/performance/${{ parameters.projectFileName }} - - ${{ if eq(parameters.osGroup, 'windows') }}: - - name: Python + - name: Python + - ${{ if eq(parameters.osGroup, 'windows') }}: value: 'py -3' - - ${{ if ne(parameters.osGroup, 'windows') }}: - - name: Python + - ${{ else }}: value: 'python3' - ${{ each variable in parameters.variables }}: - ${{ variable }} From 9d9f910423eff607049926938706668d46d6fba4 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Fri, 31 Jan 2025 08:26:36 +1000 Subject: [PATCH 44/45] Fix default parameter values for pipeline --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c37e291adb7..782d00f1209 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -58,12 +58,12 @@ jobs: - template: /eng/pipelines/sdk-perf-jobs.yml parameters: ${{ if or(eq(variables['System.TeamProject'], 'public'), parameters.runPublicJobs) }}: - runPublicJobs: ${{ parameters.runPublicJobs }} + runPublicJobs: true ${{ if ne(variables['System.TeamProject'], 'public') }}: ${{ if or(notin(variables['Build.Reason'], 'PullRequest', 'Manual'), parameters.runPrivateJobs) }}: - runPrivateJobs: ${{ parameters.runPrivateJobs }} + runPrivateJobs: true ${{ if or(eq(variables['Build.CronSchedule.DisplayName'], 'Every 12 hours build'), parameters.runScheduledPrivateJobs) }}: - runScheduledPrivateJobs: ${{ parameters.runScheduledPrivateJobs }} + runScheduledPrivateJobs: true jobParameters: ${{ if parameters.onlySanityCheck }}: onlySanityCheck: true From ac8d7088387fb089e1a66eb93dfc317ad6a3f3b2 Mon Sep 17 00:00:00 2001 From: Cameron Aavik Date: Fri, 31 Jan 2025 08:48:41 +1000 Subject: [PATCH 45/45] Fix if else condition for python variable --- eng/pipelines/templates/run-performance-job.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index e50a71ef251..b9266adcf21 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -129,9 +129,9 @@ jobs: ${{ else }}: value: $(performanceRepoDir)/eng/performance/${{ parameters.projectFileName }} - name: Python - - ${{ if eq(parameters.osGroup, 'windows') }}: + ${{ if eq(parameters.osGroup, 'windows') }}: value: 'py -3' - - ${{ else }}: + ${{ else }}: value: 'python3' - ${{ each variable in parameters.variables }}: - ${{ variable }}