From 055e09d0262ca26ab15967a7db2da15d78d6592e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 26 Aug 2024 13:54:38 +0200 Subject: [PATCH 1/9] SPMI: Add a public test version of the superpmi-collect pipeline This one does all the collection steps except it does not upload the collection. --- .../templates/runtimes/run-test-job.yml | 20 +- .../coreclr/superpmi-collect-test.yml | 371 ++++++++++++++++++ eng/pipelines/coreclr/superpmi-collect.yml | 1 + .../templates/run-superpmi-collect-job.yml | 18 +- .../templates/superpmi-collect-job.yml | 2 + eng/pipelines/libraries/run-test-job.yml | 2 + .../libraries/superpmi-postprocess-step.yml | 20 +- 7 files changed, 408 insertions(+), 26 deletions(-) create mode 100644 eng/pipelines/coreclr/superpmi-collect-test.yml diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 38088e51f405f2..2d8bc4e937b8a9 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -23,6 +23,7 @@ parameters: shouldContinueOnError: false dependsOn: [] SuperPmiCollect: false + SuperPmiCollectionUpload: true unifiedArtifactsName: '' unifiedBuildNameSuffix: '' unifiedBuildConfigOverride: '' @@ -622,15 +623,16 @@ jobs: displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages condition: always() - - task: AzureCLI@2 - displayName: 'Upload SuperPMI $(CollectionName)-$(CollectionType) collection to Azure Storage' - inputs: - azureSubscription: 'superpmi-collect-rw' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper) - condition: always() + - ${{ if eq(parameters.SuperPmiCollectionUpload, true) }}: + - task: AzureCLI@2 + displayName: 'Upload SuperPMI $(CollectionName)-$(CollectionType) collection to Azure Storage' + inputs: + azureSubscription: 'superpmi-collect-rw' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper) + condition: always() - task: CopyFiles@2 displayName: Copying superpmi.log of all partitions diff --git a/eng/pipelines/coreclr/superpmi-collect-test.yml b/eng/pipelines/coreclr/superpmi-collect-test.yml new file mode 100644 index 00000000000000..dc9408c30a47d5 --- /dev/null +++ b/eng/pipelines/coreclr/superpmi-collect-test.yml @@ -0,0 +1,371 @@ +# This job definition automates the SuperPMI collection process. + +trigger: none + +variables: + - template: /eng/pipelines/common/variables.yml + +extends: + template: /eng/pipelines/common/templates/pipeline-with-resources.yml + parameters: + stages: + - stage: Build + jobs: + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: + - windows_x64 + - linux_x64 + jobParameters: + testGroup: outerloop + buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true + timeoutInMinutes: 120 + postBuildSteps: + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop + disableComponentGovernance: true # No shipping artifacts produced by this pipeline + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: + - windows_x86 + - windows_arm64 + - osx_arm64 + jobParameters: + testGroup: outerloop + buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true + timeoutInMinutes: 120 + postBuildSteps: + # Build CLR assets for x64 as well as the target as we need an x64 mcs + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.spmi -c $(_BuildConfig) + archParameter: -arch x64 + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop + disableComponentGovernance: true # No shipping artifacts produced by this pipeline + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: checked + platforms: + - linux_arm + - linux_arm64 + jobParameters: + testGroup: outerloop + buildArgs: -s clr+libs+libs.tests -rc $(_BuildConfig) -c Release /p:ArchiveTests=true + timeoutInMinutes: 120 + postBuildSteps: + # Build CLR assets for x64 as well as the target as we need an x64 mcs + - template: /eng/pipelines/common/templates/global-build-step.yml + parameters: + buildArgs: -s clr.spmi -c $(_BuildConfig) + archParameter: -arch x64 + container: linux_x64 + - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/bin + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + - template: /eng/pipelines/common/upload-artifact-step.yml + parameters: + rootFolder: $(Build.SourcesDirectory)/artifacts/helix + includeRootFolder: false + archiveType: $(archiveType) + archiveExtension: $(archiveExtension) + tarCompression: $(tarCompression) + artifactName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + extraVariablesTemplates: + - template: /eng/pipelines/common/templates/runtimes/native-test-assets-variables.yml + parameters: + testGroup: outerloop + disableComponentGovernance: true # No shipping artifacts produced by this pipeline + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml + buildConfig: checked + platforms: + - CoreClrTestBuildHost # Either osx_x64 or linux_x64 + jobParameters: + testGroup: outerloop + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: pmi + collectionName: libraries + collectionUpload: false + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: crossgen2 + collectionName: libraries + collectionUpload: false + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: run + collectionName: realworld + collectionUpload: false + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: run + collectionName: benchmarks + collectionUpload: false + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: run_tiered + collectionName: benchmarks + collectionUpload: false + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: run_pgo + collectionName: benchmarks + collectionUpload: false + + # + # Collection of coreclr test run + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/templates/runtimes/run-test-job.yml + buildConfig: checked + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: superpmi + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + SuperPmiCollect: true + SuperPmiCollectionUpload: false + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_$(_BuildConfig) + + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/superpmi-collect-job.yml + buildConfig: checked + platforms: + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_arm64 + helixQueueGroup: ci + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + collectionType: nativeaot + collectionName: smoke_tests + collectionUpload: false + + # + # Collection of libraries test run: normal + # Libraries Test Run using Release libraries, and Checked CoreCLR + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/run-test-job.yml + buildConfig: Release + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: superpmi + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testScope: innerloop + liveRuntimeBuildConfig: Checked + dependsOnTestBuildConfiguration: Release + dependsOnTestArchitecture: x64 + scenarios: + - normal + SuperPmiCollect: true + SuperPmiCollectionName: libraries_tests + SuperPmiCollectionUpload: false + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + unifiedBuildConfigOverride: checked + + # + # Collection of libraries test run: no_tiered_compilation + # Libraries Test Run using Release libraries, and Checked CoreCLR + # + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/run-test-job.yml + buildConfig: Release + platforms: + - osx_arm64 + - linux_arm + - linux_arm64 + - linux_x64 + - windows_x64 + - windows_x86 + - windows_arm64 + helixQueueGroup: superpmi + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testScope: innerloop + liveRuntimeBuildConfig: Checked + dependsOnTestBuildConfiguration: Release + dependsOnTestArchitecture: x64 + scenarios: + - no_tiered_compilation + SuperPmiCollect: true + SuperPmiCollectionName: libraries_tests_no_tiered_compilation + SuperPmiCollectionUpload: false + unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked + unifiedBuildConfigOverride: checked diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index ea5b57d5f51df2..cc229fdc766803 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -377,6 +377,7 @@ extends: - no_tiered_compilation SuperPmiCollect: true SuperPmiCollectionName: libraries_tests_no_tiered_compilation + SuperPmiCollectionUpload: true unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked unifiedBuildConfigOverride: checked diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 3ef32f30b2d415..3b3c420823ae57 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -16,6 +16,7 @@ parameters: liveLibrariesBuildConfig: '' # optional -- live-live libraries configuration to use for the run collectionType: '' collectionName: '' + collectionUpload: true jobs: - template: /eng/pipelines/common/templates/runtimes/xplat-job.yml @@ -181,14 +182,15 @@ jobs: - script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall && $(PipScript) install azure.identity==1.16.1 --force-reinstall displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages - - task: AzureCLI@2 - displayName: ${{ format('Upload SuperPMI {0}-{1} collection to Azure Storage', parameters.collectionName, parameters.collectionType) }} - inputs: - azureSubscription: 'superpmi-collect-rw' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper) + - ${{ if eq(parameters.collectionUpload, true) }}: + - task: AzureCLI@2 + displayName: ${{ format('Upload SuperPMI {0}-{1} collection to Azure Storage', parameters.collectionName, parameters.collectionType) }} + inputs: + azureSubscription: 'superpmi-collect-rw' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch $(archType) -build_type $(buildConfig) -mch_files $(MergedMchFileLocation)$(CollectionName).$(CollectionType).$(MchFileTag).mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).x64.$(buildConfigUpper) # Always upload the available logs for diagnostics - task: CopyFiles@2 diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml index 6af16c3db87642..23eb5c199f2f84 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml @@ -9,6 +9,7 @@ parameters: runJobTemplate: '/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml' collectionType: '' collectionName: '' + collectionUpload: true ### SuperPMI job ### Each collection job depends on a corresponding build job with the same buildConfig and archType. @@ -27,6 +28,7 @@ jobs: liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} collectionType: ${{ parameters.collectionType }} collectionName: ${{ parameters.collectionName }} + collectionUpload: ${{ parameters.collectionUpload }} # Test job depends on the corresponding build job dependsOn: - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_' diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index 82017494a4bb5b..f42eb96061788d 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -21,6 +21,7 @@ parameters: SuperPmiCollect: false SuperPmiCollectionType: 'run' SuperPmiCollectionName: 'libraries_tests' + SuperPmiCollectionUpload: true dependsOn: [] unifiedArtifactsName: '' helixArtifactsName: '' @@ -162,6 +163,7 @@ jobs: archType: ${{ parameters.archType }} SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} + SuperPmiCollectionUpload: ${{ parameters.SuperPmiCollectionUpload }} MergedMchFileLocation: $(MergedMchFileLocation) MchFilesLocation: $(MchFilesLocation) SpmiLogsLocation: $(SpmiLogsLocation) diff --git a/eng/pipelines/libraries/superpmi-postprocess-step.yml b/eng/pipelines/libraries/superpmi-postprocess-step.yml index 343cb1ee58c118..10aea3041c540e 100644 --- a/eng/pipelines/libraries/superpmi-postprocess-step.yml +++ b/eng/pipelines/libraries/superpmi-postprocess-step.yml @@ -14,6 +14,7 @@ parameters: archType: '' SuperPmiCollectionType: 'run' SuperPmiCollectionName: 'libraries_tests' + SuperPmiCollectionUpload: true MergedMchFileLocation: '' MchFilesLocation: '' SpmiLogsLocation: '' @@ -70,15 +71,16 @@ steps: displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages condition: always() - - task: AzureCLI@2 - displayName: 'Upload SuperPMI ${{ parameters.SuperPmiCollectionName }}-${{ parameters.SuperPmiCollectionType }} collection to Azure Storage' - inputs: - azureSubscription: 'superpmi-collect-rw' - scriptType: 'pscore' - scriptLocation: 'inlineScript' - inlineScript: | - ${{ parameters.PythonScript }} $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch ${{ parameters.archType }} -build_type ${{ parameters.buildConfig }} -mch_files ${{ parameters.MergedMchFileLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}.${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}.mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.x64.${{ parameters.buildConfigUpper }} - condition: always() + - ${{ if eq(parameters.SuperPmiCollectionUpload, true) }}: + - task: AzureCLI@2 + displayName: 'Upload SuperPMI ${{ parameters.SuperPmiCollectionName }}-${{ parameters.SuperPmiCollectionType }} collection to Azure Storage' + inputs: + azureSubscription: 'superpmi-collect-rw' + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + ${{ parameters.PythonScript }} $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi.py upload -log_level DEBUG -arch ${{ parameters.archType }} -build_type ${{ parameters.buildConfig }} -mch_files ${{ parameters.MergedMchFileLocation }}${{ parameters.SuperPmiCollectionName }}.${{ parameters.SuperPmiCollectionType }}.${{ parameters.osGroup }}.${{ parameters.archType }}.${{ parameters.buildConfig }}.mch -core_root $(Build.SourcesDirectory)/artifacts/bin/coreclr/${{ parameters.osGroup }}.x64.${{ parameters.buildConfigUpper }} + condition: always() - task: CopyFiles@2 displayName: Copying superpmi.log of all partitions From 0e28bc10cc8315a947946b487dc7618d58bd5dcf Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 26 Aug 2024 16:29:26 +0200 Subject: [PATCH 2/9] Add creator --- eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 3b3c420823ae57..87fab6732a5528 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -140,7 +140,10 @@ jobs: HelixAccessToken: $(HelixApiAccessToken) HelixTargetQueues: $(Queue) HelixPreCommands: $(HelixPreCommand) - Creator: $(Creator) + + ${{ if eq(variables['System.TeamProject'], 'public') }}: + Creator: $(Build.DefinitionName) + WorkItemTimeout: 4:00 # 4 hours WorkItemDirectory: '$(WorkItemDirectory)' CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' From 67592b968de1283662e6030147c4c32e0548462b Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 27 Aug 2024 10:51:30 +0200 Subject: [PATCH 3/9] Hopefully fix --- eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml | 5 +---- eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 87fab6732a5528..6fba148b83f30b 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -140,10 +140,7 @@ jobs: HelixAccessToken: $(HelixApiAccessToken) HelixTargetQueues: $(Queue) HelixPreCommands: $(HelixPreCommand) - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - Creator: $(Build.DefinitionName) - + Creator: $(Build.DefinitionName) WorkItemTimeout: 4:00 # 4 hours WorkItemDirectory: '$(WorkItemDirectory)' CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)' diff --git a/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml b/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml index 2a85d76c07885b..0d0a26c132af49 100644 --- a/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml +++ b/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml @@ -56,5 +56,5 @@ steps: DotNetCliVersion: ${{ parameters.DotNetCliVersion }} EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - Creator: ${{ parameters.Creator }} + _Creator: ${{ parameters.Creator }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) From 5693b65f7917cc4338fa39e788dd2099a70ac555 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 27 Aug 2024 12:05:26 +0200 Subject: [PATCH 4/9] Hopefully fix 2 --- eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml | 2 +- src/coreclr/scripts/superpmi-collect.proj | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml b/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml index 0d0a26c132af49..daed124df41593 100644 --- a/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml +++ b/eng/pipelines/coreclr/templates/superpmi-send-to-helix.yml @@ -35,6 +35,7 @@ steps: condition: ${{ parameters.condition }} shouldContinueOnError: ${{ parameters.continueOnError }} environment: + _Creator: ${{ parameters.Creator }} TargetOS: ${{ parameters.osGroup }} TargetArchitecture: ${{ parameters.archType }} MchFileTag: $(MchFileTag) @@ -56,5 +57,4 @@ steps: DotNetCliVersion: ${{ parameters.DotNetCliVersion }} EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} - _Creator: ${{ parameters.Creator }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj index 2b15fac5698134..00d0a89da206b9 100644 --- a/src/coreclr/scripts/superpmi-collect.proj +++ b/src/coreclr/scripts/superpmi-collect.proj @@ -148,6 +148,7 @@ false false + $(_Creator) From 6134a4f4d332e9dfa9e0ab6c42f295e168723cfb Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 27 Aug 2024 13:58:24 +0200 Subject: [PATCH 5/9] Use public queues, switch to System.TeamProject env var --- .../templates/runtimes/run-test-job.yml | 3 +- eng/pipelines/coreclr/superpmi-collect.yml | 1 - .../templates/run-superpmi-collect-job.yml | 11 +++-- .../templates/superpmi-collect-job.yml | 2 - eng/pipelines/libraries/run-test-job.yml | 2 - .../libraries/superpmi-postprocess-step.yml | 3 +- src/coreclr/scripts/superpmi_collect_setup.py | 42 +++++++++++++------ 7 files changed, 40 insertions(+), 24 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 2d8bc4e937b8a9..0dbe3d14bb665d 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -23,7 +23,6 @@ parameters: shouldContinueOnError: false dependsOn: [] SuperPmiCollect: false - SuperPmiCollectionUpload: true unifiedArtifactsName: '' unifiedBuildNameSuffix: '' unifiedBuildConfigOverride: '' @@ -623,7 +622,7 @@ jobs: displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages condition: always() - - ${{ if eq(parameters.SuperPmiCollectionUpload, true) }}: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - task: AzureCLI@2 displayName: 'Upload SuperPMI $(CollectionName)-$(CollectionType) collection to Azure Storage' inputs: diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml index cc229fdc766803..ea5b57d5f51df2 100644 --- a/eng/pipelines/coreclr/superpmi-collect.yml +++ b/eng/pipelines/coreclr/superpmi-collect.yml @@ -377,7 +377,6 @@ extends: - no_tiered_compilation SuperPmiCollect: true SuperPmiCollectionName: libraries_tests_no_tiered_compilation - SuperPmiCollectionUpload: true unifiedArtifactsName: BuildArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked helixArtifactsName: LibrariesTestArtifacts_$(osGroup)$(osSubgroup)_$(archType)_Checked unifiedBuildConfigOverride: checked diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index 6fba148b83f30b..ba66ff8e8ee603 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -16,7 +16,6 @@ parameters: liveLibrariesBuildConfig: '' # optional -- live-live libraries configuration to use for the run collectionType: '' collectionName: '' - collectionUpload: true jobs: - template: /eng/pipelines/common/templates/runtimes/xplat-job.yml @@ -105,6 +104,12 @@ jobs: - name: InputDirectory value: '$(Build.SourcesDirectory)/artifacts/tests/coreclr/obj/${{ parameters.osGroup }}.${{ parameters.archType }}.$(buildConfigUpper)/Managed/nativeaot/SmokeTests' + - name: PublicQueuesCLIArg + ${{ if ne(variables['System.TeamProject'], 'internal') }}: + value: '-public_queues' + ${{ else }}: + value: '' + workspace: clean: all pool: @@ -117,7 +122,7 @@ jobs: displayName: Enable python venv condition: always() - - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 25 # size in MB + - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_collect_setup.py -payload_directory $(PayloadLocation) -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -platform $(osGroup) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) $(PublicQueuesCLIArg) -max_size 25 # size in MB displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }} # Create required directories for merged mch collection and superpmi logs @@ -182,7 +187,7 @@ jobs: - script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall && $(PipScript) install azure.identity==1.16.1 --force-reinstall displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages - - ${{ if eq(parameters.collectionUpload, true) }}: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - task: AzureCLI@2 displayName: ${{ format('Upload SuperPMI {0}-{1} collection to Azure Storage', parameters.collectionName, parameters.collectionType) }} inputs: diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml index 23eb5c199f2f84..6af16c3db87642 100644 --- a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml @@ -9,7 +9,6 @@ parameters: runJobTemplate: '/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml' collectionType: '' collectionName: '' - collectionUpload: true ### SuperPMI job ### Each collection job depends on a corresponding build job with the same buildConfig and archType. @@ -28,7 +27,6 @@ jobs: liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} collectionType: ${{ parameters.collectionType }} collectionName: ${{ parameters.collectionName }} - collectionUpload: ${{ parameters.collectionUpload }} # Test job depends on the corresponding build job dependsOn: - 'build_${{ parameters.osGroup }}${{ parameters.osSubgroup }}_${{ parameters.archType }}_${{ parameters.buildConfig }}_' diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml index f42eb96061788d..82017494a4bb5b 100644 --- a/eng/pipelines/libraries/run-test-job.yml +++ b/eng/pipelines/libraries/run-test-job.yml @@ -21,7 +21,6 @@ parameters: SuperPmiCollect: false SuperPmiCollectionType: 'run' SuperPmiCollectionName: 'libraries_tests' - SuperPmiCollectionUpload: true dependsOn: [] unifiedArtifactsName: '' helixArtifactsName: '' @@ -163,7 +162,6 @@ jobs: archType: ${{ parameters.archType }} SuperPmiCollectionType: ${{ parameters.SuperPmiCollectionType }} SuperPmiCollectionName: ${{ parameters.SuperPmiCollectionName }} - SuperPmiCollectionUpload: ${{ parameters.SuperPmiCollectionUpload }} MergedMchFileLocation: $(MergedMchFileLocation) MchFilesLocation: $(MchFilesLocation) SpmiLogsLocation: $(SpmiLogsLocation) diff --git a/eng/pipelines/libraries/superpmi-postprocess-step.yml b/eng/pipelines/libraries/superpmi-postprocess-step.yml index 10aea3041c540e..7412eba9db5474 100644 --- a/eng/pipelines/libraries/superpmi-postprocess-step.yml +++ b/eng/pipelines/libraries/superpmi-postprocess-step.yml @@ -14,7 +14,6 @@ parameters: archType: '' SuperPmiCollectionType: 'run' SuperPmiCollectionName: 'libraries_tests' - SuperPmiCollectionUpload: true MergedMchFileLocation: '' MchFilesLocation: '' SpmiLogsLocation: '' @@ -71,7 +70,7 @@ steps: displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages condition: always() - - ${{ if eq(parameters.SuperPmiCollectionUpload, true) }}: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - task: AzureCLI@2 displayName: 'Upload SuperPMI ${{ parameters.SuperPmiCollectionName }}-${{ parameters.SuperPmiCollectionType }} collection to Azure Storage' inputs: diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 40c3a768e68283..d71ea5d60f8b75 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -53,6 +53,7 @@ parser.add_argument("-mch_file_tag", help="Tag to be used to mch files") parser.add_argument("-input_directory", help="Directory containing assemblies which SuperPMI will use for collection (for pmi/crossgen2 collections)") parser.add_argument("-max_size", help="Max size of each partition in MB (for pmi/crossgen2 collections)") +parser.add_argument("-public_queues", action="store_true", help="Whether to set up for public queues (or internal ones, if not specified)") is_windows = platform.system() == "Windows" @@ -264,6 +265,11 @@ def setup_args(args): max_size) * 1000 * 1000 if max_size is not None and max_size.isnumeric() else 0 # Convert to MB ) + + coreclr_args.verify(args, + "public_queues", + lambda unused: True, + "Whether to use public queues (or, if not specified, internal queues)") return coreclr_args @@ -451,21 +457,33 @@ def main(main_args): platform_name = coreclr_args.platform.lower() helix_source_prefix = "official" creator = "" - ci = True # Determine the Helix queue name to use when running jobs. # Note that we run in the 'internal', not 'public', instance, so we must use 'internal' queues defined in helix-queues-setup.yml. - if platform_name == "windows": - helix_queue = "Windows.11.Arm64" if arch == "arm64" else "Windows.10.Amd64.X86.Rt" - elif platform_name == "linux": - if arch == "arm": - helix_queue = "(Debian.12.Arm32)Ubuntu.2004.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-helix-arm32v7" - elif arch == "arm64": - helix_queue = "(Ubuntu.1804.Arm64)Ubuntu.2004.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8" - else: - helix_queue = "Ubuntu.2204.Amd64" - elif platform_name == "osx": - helix_queue = "OSX.1200.ARM64" if arch == "arm64" else "OSX.1200.Amd64" + if coreclr_args.public_queues: + if platform_name == "windows": + helix_queue = "Windows.11.Arm64.Open" if arch == "arm64" else "Windows.10.Amd64.Open" + elif platform_name == "linux": + if arch == "arm": + helix_queue = "(Debian.12.Arm32.Open)Ubuntu.2004.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-helix-arm32v7" + elif arch == "arm64": + helix_queue = "(Ubuntu.1804.Arm64.Open)Ubuntu.2004.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8" + else: + helix_queue = "Ubuntu.2204.Amd64.Open" + elif platform_name == "osx": + helix_queue = "OSX.1200.ARM64.Open" if arch == "arm64" else "OSX.1200.Amd64.Open" + else: + if platform_name == "windows": + helix_queue = "Windows.11.Arm64" if arch == "arm64" else "Windows.10.Amd64.X86.Rt" + elif platform_name == "linux": + if arch == "arm": + helix_queue = "(Debian.12.Arm32)Ubuntu.2004.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-helix-arm32v7" + elif arch == "arm64": + helix_queue = "(Ubuntu.1804.Arm64)Ubuntu.2004.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8" + else: + helix_queue = "Ubuntu.2204.Amd64" + elif platform_name == "osx": + helix_queue = "OSX.1200.ARM64" if arch == "arm64" else "OSX.1200.Amd64" # Copy the superpmi scripts From 40d8d1edb20ffe5604f75a4ff80b746f550c2e6e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 27 Aug 2024 17:24:29 +0200 Subject: [PATCH 6/9] Remove outdated comment --- src/coreclr/scripts/superpmi_collect_setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index d71ea5d60f8b75..06c0f7c4785890 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -459,7 +459,6 @@ def main(main_args): creator = "" # Determine the Helix queue name to use when running jobs. - # Note that we run in the 'internal', not 'public', instance, so we must use 'internal' queues defined in helix-queues-setup.yml. if coreclr_args.public_queues: if platform_name == "windows": helix_queue = "Windows.11.Arm64.Open" if arch == "arm64" else "Windows.10.Amd64.Open" From 24451b8262d27c0e8312dca9bca6b88e414a0d1e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 27 Aug 2024 17:26:50 +0200 Subject: [PATCH 7/9] Run Azure package stuff internally only --- .../templates/runtimes/run-test-job.yml | 24 +++++++++---------- .../templates/run-superpmi-collect-job.yml | 20 ++++++++-------- .../libraries/superpmi-postprocess-step.yml | 24 +++++++++---------- src/coreclr/scripts/superpmi_collect_setup.py | 2 -- 4 files changed, 34 insertions(+), 36 deletions(-) diff --git a/eng/pipelines/common/templates/runtimes/run-test-job.yml b/eng/pipelines/common/templates/runtimes/run-test-job.yml index 0dbe3d14bb665d..de05dba7078259 100644 --- a/eng/pipelines/common/templates/runtimes/run-test-job.yml +++ b/eng/pipelines/common/templates/runtimes/run-test-job.yml @@ -609,20 +609,20 @@ jobs: displayName: 'Upload artifacts SuperPMI $(CollectionName)-$(CollectionType) collection' condition: always() - # Add authenticated pip feed - - task: PipAuthenticate@1 - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: public/dotnet-public-pypi - onlyAddExtraIndex: false - condition: always() + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + # Add authenticated pip feed + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: public/dotnet-public-pypi + onlyAddExtraIndex: false + condition: always() - # Ensure the Python azure-storage-blob package is installed before doing the upload. - - script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall && $(PipScript) install azure.identity==1.16.1 --force-reinstall - displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages - condition: always() + # Ensure the Python azure-storage-blob package is installed before doing the upload. + - script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall && $(PipScript) install azure.identity==1.16.1 --force-reinstall + displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages + condition: always() - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - task: AzureCLI@2 displayName: 'Upload SuperPMI $(CollectionName)-$(CollectionType) collection to Azure Storage' inputs: diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index ba66ff8e8ee603..d5367d0c47e7ed 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -176,18 +176,18 @@ jobs: artifactName: 'SuperPMI_Collection_$(CollectionName)_$(CollectionType)_$(osGroup)$(osSubgroup)_$(archType)_$(buildConfig)' displayName: ${{ format('Upload artifacts SuperPMI {0}-{1} collection', parameters.collectionName, parameters.collectionType) }} - # Add authenticated pip feed - - task: PipAuthenticate@1 - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: public/dotnet-public-pypi - onlyAddExtraIndex: false + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + # Add authenticated pip feed + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: public/dotnet-public-pypi + onlyAddExtraIndex: false - # Ensure the Python azure-storage-blob package is installed before doing the upload. - - script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall && $(PipScript) install azure.identity==1.16.1 --force-reinstall - displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages + # Ensure the Python azure-storage-blob package is installed before doing the upload. + - script: $(PipScript) install --upgrade pip && $(PipScript) install azure.storage.blob==12.5.0 --force-reinstall && $(PipScript) install azure.identity==1.16.1 --force-reinstall + displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - task: AzureCLI@2 displayName: ${{ format('Upload SuperPMI {0}-{1} collection to Azure Storage', parameters.collectionName, parameters.collectionType) }} inputs: diff --git a/eng/pipelines/libraries/superpmi-postprocess-step.yml b/eng/pipelines/libraries/superpmi-postprocess-step.yml index 7412eba9db5474..5ee73194e463e6 100644 --- a/eng/pipelines/libraries/superpmi-postprocess-step.yml +++ b/eng/pipelines/libraries/superpmi-postprocess-step.yml @@ -57,20 +57,20 @@ steps: displayName: 'Upload artifacts SuperPMI ${{ parameters.SuperPmiCollectionName }}-${{ parameters.SuperPmiCollectionType }} collection' condition: always() - # Add authenticated pip feed - - task: PipAuthenticate@1 - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: public/dotnet-public-pypi - onlyAddExtraIndex: false - condition: always() + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + # Add authenticated pip feed + - task: PipAuthenticate@1 + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: public/dotnet-public-pypi + onlyAddExtraIndex: false + condition: always() - # Ensure the Python azure-storage-blob package is installed before doing the upload. - - script: ${{ parameters.PipScript }} install --upgrade pip && ${{ parameters.PipScript }} install azure.storage.blob==12.5.0 --force-reinstall && ${{ parameters.PipScript }} install azure.identity==1.16.1 --force-reinstall - displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages - condition: always() + # Ensure the Python azure-storage-blob package is installed before doing the upload. + - script: ${{ parameters.PipScript }} install --upgrade pip && ${{ parameters.PipScript }} install azure.storage.blob==12.5.0 --force-reinstall && ${{ parameters.PipScript }} install azure.identity==1.16.1 --force-reinstall + displayName: Upgrade Pip to latest and install azure-storage-blob and azure-identity Python packages + condition: always() - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - task: AzureCLI@2 displayName: 'Upload SuperPMI ${{ parameters.SuperPmiCollectionName }}-${{ parameters.SuperPmiCollectionType }} collection to Azure Storage' inputs: diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 06c0f7c4785890..1486916f367e28 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -456,7 +456,6 @@ def main(main_args): arch = coreclr_args.arch platform_name = coreclr_args.platform.lower() helix_source_prefix = "official" - creator = "" # Determine the Helix queue name to use when running jobs. if coreclr_args.public_queues: @@ -624,7 +623,6 @@ def make_readable(folder_name): set_pipeline_variable("InputArtifacts", input_artifacts) set_pipeline_variable("Python", ' '.join(get_python_name())) set_pipeline_variable("Architecture", arch) - set_pipeline_variable("Creator", creator) set_pipeline_variable("Queue", helix_queue) set_pipeline_variable("HelixSourcePrefix", helix_source_prefix) set_pipeline_variable("MchFileTag", coreclr_args.mch_file_tag) From 426c5272716bd06b748f87e4d0ca72ab6906e0f4 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 27 Aug 2024 20:34:22 +0200 Subject: [PATCH 8/9] Fix linux-arm64 queue --- src/coreclr/scripts/superpmi_collect_setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/scripts/superpmi_collect_setup.py b/src/coreclr/scripts/superpmi_collect_setup.py index 1486916f367e28..c1e325e8daeac2 100644 --- a/src/coreclr/scripts/superpmi_collect_setup.py +++ b/src/coreclr/scripts/superpmi_collect_setup.py @@ -465,7 +465,7 @@ def main(main_args): if arch == "arm": helix_queue = "(Debian.12.Arm32.Open)Ubuntu.2004.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-12-helix-arm32v7" elif arch == "arm64": - helix_queue = "(Ubuntu.1804.Arm64.Open)Ubuntu.2004.ArmArch@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-helix-arm64v8" + helix_queue = "(Ubuntu.2004.Arm64.Open)Ubuntu.2004.Armarch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04-helix-arm64v8" else: helix_queue = "Ubuntu.2204.Amd64.Open" elif platform_name == "osx": From 8e8694b3ebc3ff8de2b1c9f9e7075d8900cb71f8 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 28 Aug 2024 15:38:30 +0200 Subject: [PATCH 9/9] Do not specify Creator for internal work items, which is illegal --- eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml index d5367d0c47e7ed..9bf2ed9d5bceee 100644 --- a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml +++ b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml @@ -145,7 +145,10 @@ jobs: HelixAccessToken: $(HelixApiAccessToken) HelixTargetQueues: $(Queue) HelixPreCommands: $(HelixPreCommand) - Creator: $(Build.DefinitionName) + + ${{ if ne(variables['System.TeamProject'], 'internal') }}: + Creator: $(Build.DefinitionName) + WorkItemTimeout: 4:00 # 4 hours WorkItemDirectory: '$(WorkItemDirectory)' CorrelationPayloadDirectory: '$(CorrelationPayloadDirectory)'