From 8956449304180ee461c61385d7d6b6587f09541f Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 12:48:43 +0100 Subject: [PATCH 01/34] 1ES pipelines migration --- azure-pipelines.yml | 1029 ++++++++----------------------------------- 1 file changed, 178 insertions(+), 851 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a3f9bb5318..601c1ebb99 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -3,7 +3,6 @@ trigger: branches: include: - main - - dev16.1 - feature/* - release/* paths: @@ -25,34 +24,19 @@ trigger: - release-notes.md - TESTGUIDE.md -pr: - branches: - include: - - main - - dev16.1 - - feature/* - - release/* - paths: - include: - - '*' - exclude: - - .github/* - - docs/ - - attributions.md - - CODE_OF_CONDUCT.md - - DEVGUIDE.md - - INTERNAL.md - - Language-Version-History.md - - License.txt - - README.md - - release-notes.md - - TESTGUIDE.md - variables: + - name: ReleaseBranch + value: release/dev17.9 + - name: VSTargetInsertionBranch + value: main - name: _TeamName value: FSharp + - name: TeamName + value: FSharp - name: _BuildConfig value: Release + - name: _SignType + value: Real - name: _PublishUsingPipelines value: true - name: _DotNetArtifactsCategory @@ -61,833 +45,176 @@ variables: value: Products/$(System.TeamProject)/$(Build.Repository.Name)/$(Build.SourceBranchName)/$(Build.BuildNumber) - name: Codeql.Enabled value: true - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: _DotNetValidationArtifactsCategory - value: .NETCoreValidation - - group: DotNet-FSharp-SDLValidation-Params - - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: - - name: RunningAsPullRequest - value: true - # Pick up pool provider name behavior from shared yaml template - - template: /eng/common/templates/variables/pool-providers.yml - -# Variables defined in yml cannot be overridden at queue time; instead overridable variables must be defined in the web UI. -# Commenting out until something like this is supported: https://github.com/Microsoft/azure-pipelines-yaml/pull/129 -#variables: -#- name: SkipTests -# defaultValue: false - -stages: -- stage: build - displayName: Build - jobs: - - #-------------------------------------------------------------------------------------------------------------------# - # Signed build # - #-------------------------------------------------------------------------------------------------------------------# - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/release/dev17.9') }}: - - template: /eng/common/templates/job/onelocbuild.yml + - name: _DotNetValidationArtifactsCategory + value: .NETCoreValidation + - group: DotNet-FSharp-SDLValidation-Params + - template: /eng/common/templates-official/variables/pool-providers.yml@self + +resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release + +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + sdl: + sourceAnalysisPool: + name: $(DncEngInternalBuildPool) + image: 1es-windows-2022-pt + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: build + displayName: Build + jobs: + #-------------------------------------------------------------------------------------------------------------------# + # Signed build # + #-------------------------------------------------------------------------------------------------------------------# + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/release/dev17.9') }}: + - template: /eng/common/templates-official/job/onelocbuild.yml@self + parameters: + MirrorRepo: fsharp + MirrorBranch: release/dev17.9 + LclSource: lclFilesfromPackage + LclPackageId: 'LCL-JUNO-PROD-FSHARP' + - template: /eng/common/templates-official/jobs/jobs.yml@self parameters: - MirrorRepo: fsharp - MirrorBranch: release/dev17.9 - LclSource: lclFilesfromPackage - LclPackageId: 'LCL-JUNO-PROD-FSHARP' - - template: /eng/common/templates/jobs/jobs.yml + enableMicrobuild: true + enablePublishBuildArtifacts: true + enablePublishTestResults: false + enablePublishBuildAssets: true + enablePublishUsingPipelines: $(_PublishUsingPipelines) + enableSourceBuild: true + enableTelemetry: true + helixRepo: dotnet/fsharp + jobs: + - job: Full_Signed + pool: + name: $(DncEngInternalBuildPool) + demands: 1es-windows-2022-pt + os: windows + timeoutInMinutes: 300 + variables: + - group: DotNet-Symbol-Server-Pats + - group: DotNet-DevDiv-Insertion-Workflow-Variables + - name: _SignType + value: Real + steps: + - checkout: self + clean: true + - template: /eng/restore-internal-tools.yml + - script: eng\CIBuild.cmd + -configuration $(_BuildConfig) + -prepareMachine + -testAllButIntegration + -officialSkipTests $(SkipTests) + /p:SignType=$(_SignType) + /p:DotNetSignType=$(_SignType) + /p:MicroBuild_SigningEnabled=true + /p:OverridePackageSource=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + /p:PublishToSymbolServer=true + /p:VisualStudioDropName=$(VisualStudioDropName) + /p:GenerateSbom=true + env: + NativeToolsOnMachine: true + - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c $(_BuildConfig) + displayName: End to end build tests + continueOnError: true + condition: always() + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'NUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + continueOnError: true + condition: ne(variables['SkipTests'], 'true') + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig)' + ArtifactName: 'Test Logs' + publishLocation: Container + continueOnError: true + condition: ne(variables['SkipTests'], 'true') + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact Packages + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig)' + ArtifactName: 'Packages' + condition: succeeded() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact VSSetup + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' + ArtifactName: 'VSSetup' + condition: succeeded() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact Nightly + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpDebug.vsix' + ArtifactName: 'Nightly' + condition: succeeded() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact Symbols + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig)' + ArtifactName: 'NativeSymbols' + condition: succeeded() + - task: ms-vseng.MicroBuildTasks.4305a8de-ba66-4d8b-b2d1-0dc4ecbbf5e8.MicroBuildUploadVstsDropFolder@1 + displayName: Upload VSTS Drop + inputs: + DropName: $(VisualStudioDropName) + DropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' + AccessToken: $(dn-bot-devdiv-drop-rw-code-rw) + condition: succeeded() + + + #---------------------------------------------------------------------------------------------------------------------# + # Post Build # + #---------------------------------------------------------------------------------------------------------------------# + - template: eng/common/templates/post-build/post-build.yml parameters: - enableMicrobuild: true - enablePublishBuildArtifacts: true - enablePublishTestResults: false - enablePublishBuildAssets: true - enablePublishUsingPipelines: $(_PublishUsingPipelines) - enableSourceBuild: true - enableTelemetry: true - helixRepo: dotnet/fsharp - jobs: - - job: Full_Signed - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022preview.amd64 - timeoutInMinutes: 300 - variables: - - group: DotNet-Symbol-Server-Pats - - group: DotNet-DevDiv-Insertion-Workflow-Variables - - name: _SignType - value: Real - steps: - - checkout: self - clean: true - - template: /eng/restore-internal-tools.yml - - script: eng\CIBuild.cmd - -configuration $(_BuildConfig) - -prepareMachine - -testAllButIntegration - -officialSkipTests $(SkipTests) - /p:SignType=$(_SignType) - /p:DotNetSignType=$(_SignType) - /p:MicroBuild_SigningEnabled=true - /p:OverridePackageSource=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json - /p:TeamName=$(_TeamName) - /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) - /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) - /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - /p:PublishToSymbolServer=true - /p:VisualStudioDropName=$(VisualStudioDropName) - /p:GenerateSbom=true - env: - NativeToolsOnMachine: true - - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c $(_BuildConfig) - displayName: End to end build tests - continueOnError: true - condition: always() - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'NUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - continueOnError: true - condition: ne(variables['SkipTests'], 'true') - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig)' - ArtifactName: 'Test Logs' - publishLocation: Container - continueOnError: true - condition: ne(variables['SkipTests'], 'true') - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact Packages - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig)' - ArtifactName: 'Packages' - condition: succeeded() - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact VSSetup - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' - ArtifactName: 'VSSetup' - condition: succeeded() - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact Nightly - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpDebug.vsix' - ArtifactName: 'Nightly' - condition: succeeded() - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact Symbols - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig)' - ArtifactName: 'NativeSymbols' - condition: succeeded() - - task: ms-vseng.MicroBuildTasks.4305a8de-ba66-4d8b-b2d1-0dc4ecbbf5e8.MicroBuildUploadVstsDropFolder@1 - displayName: Upload VSTS Drop - inputs: - DropName: $(VisualStudioDropName) - DropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' - AccessToken: $(dn-bot-devdiv-drop-rw-code-rw) - condition: succeeded() - - #-------------------------------------------------------------------------------------------------------------------# - # PR builds without logs publishing # - #-------------------------------------------------------------------------------------------------------------------# - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - template: /eng/common/templates/jobs/jobs.yml + publishingInfraVersion: 3 + # Symbol validation is not entirely reliable as of yet, so should be turned off until https://github.com/dotnet/arcade/issues/2871 is resolved. + enableSymbolValidation: false + # SourceLink improperly looks for generated files. See https://github.com/dotnet/arcade/issues/3069 + enableSourceLinkValidation: false + # Enable SDL validation, passing through values from the 'DotNet-FSharp-SDLValidation-Params' group. + SDLValidationParameters: + enable: true + params: >- + -SourceToolsList @("policheck","credscan") + -ArtifactToolsList @("binskim") + -BinskimAdditionalRunConfigParams @("IgnorePdbLoadError < True","Recurse < True") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName "FSharp" + -TsaCodebaseName "FSharp-GitHub" + -TsaPublish $True + -PoliCheckAdditionalRunConfigParams @("UserExclusionPath < $(Build.SourcesDirectory)/eng/policheck_exclusions.xml") + + #---------------------------------------------------------------------------------------------------------------------# + # VS Insertion # + #---------------------------------------------------------------------------------------------------------------------# + - template: eng/release/insert-into-vs.yml parameters: - enableMicrobuild: false - enablePublishBuildArtifacts: false - enablePublishTestResults: false - enablePublishBuildAssets: false - enablePublishUsingPipelines: $(_PublishUsingPipelines) - enableSourceBuild: false - enableTelemetry: true - helixRepo: dotnet/fsharp - jobs: - # Determinism, we want to run it only in PR builds - - job: Determinism_Debug - condition: eq(variables['Build.Reason'], 'PullRequest') - variables: - - name: _SignType - value: Test - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - timeoutInMinutes: 90 - strategy: - maxParallel: 2 - matrix: - regular: - _experimental_flag: '' - experimental_features: - _experimental_flag: '' - steps: - - checkout: self - clean: true - - task: UseDotNet@2 - displayName: install SDK - inputs: - packageType: sdk - useGlobalJson: true - includePreviewVersions: false - workingDirectory: $(Build.SourcesDirectory) - installationPath: $(Build.SourcesDirectory)/.dotnet - - script: .\eng\test-determinism.cmd -configuration Debug - env: - FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) - displayName: Determinism tests with Debug configuration - - task: PublishPipelineArtifact@1 - displayName: Publish Determinism Logs - inputs: - targetPath: '$(Build.SourcesDirectory)/artifacts/log/Debug' - artifactName: 'Determinism_Debug Attempt $(System.JobAttempt) Logs' - continueOnError: true - condition: not(succeeded()) - - # Check FSComp.txt error code sorting and code formatting - - job: CheckCodeFormatting - pool: - vmImage: $(UbuntuMachineQueueName) - steps: - - checkout: self - clean: true - - script: dotnet --list-sdks - displayName: Report dotnet SDK versions - - task: UseDotNet@2 - displayName: install SDK - inputs: - packageType: sdk - useGlobalJson: true - includePreviewVersions: true - workingDirectory: $(Build.SourcesDirectory) - installationPath: $(Agent.ToolsDirectory)/dotnet - - script: dotnet tool restore - env: - DOTNET_ROLL_FORWARD_TO_PRERELEASE: 1 - displayName: Install tools - - script: dotnet fsi src/Compiler/FSCompCheck.fsx - env: - DOTNET_ROLL_FORWARD_TO_PRERELEASE: 1 - displayName: Check error code sorting in src/Compiler/FSComp.txt - - script: dotnet fantomas . --check - env: - DOTNET_ROLL_FORWARD_TO_PRERELEASE: 1 - displayName: Check code formatting (run 'dotnet fantomas .' to fix) - - # Check whether package with current version has been published to nuget.org - # We will try to restore both FSharp.Core and FCS and if restore is _successful_, package version needs to be bumped. - # NOTE: The check now runs on all branches (not just release), - # because we want to catch cases when version is desynched and we didn't update it. - # It is also helping the release notes automation to be up to date with packages versions. - - job: Check_Published_Package_Versions - # condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), or(startsWith(variables['System.PullRequest.SourceBranch'], 'release/dev'), startsWith(variables['System.PullRequest.TargetBranch'], 'release/dev'))) - pool: - vmImage: $(UbuntuMachineQueueName) - strategy: - maxParallel: 2 - matrix: - FCS: - _project: "FSharp.Compiler.Service_notshipped.fsproj" - FSCore: - _project: "FSharp.Core_notshipped.fsproj" - steps: - - checkout: self - clean: true - - task: UseDotNet@2 - displayName: install SDK - inputs: - packageType: sdk - useGlobalJson: true - includePreviewVersions: true - workingDirectory: $(Build.SourcesDirectory) - installationPath: $(Agent.ToolsDirectory)/dotnet - - pwsh: ./check.ps1 -project $(_project) - workingDirectory: $(Build.SourcesDirectory)/buildtools/checkpackages - env: - DOTNET_ROLL_FORWARD_TO_PRERELEASE: 1 - displayName: Check published package version - - - #-------------------------------------------------------------------------------------------------------------------# - # PR builds # - #-------------------------------------------------------------------------------------------------------------------# - - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - template: /eng/common/templates/jobs/jobs.yml - parameters: - enableMicrobuild: true - enablePublishBuildArtifacts: true - enablePublishTestResults: false - enablePublishBuildAssets: true - enablePublishUsingPipelines: $(_PublishUsingPipelines) - enableSourceBuild: true - enableTelemetry: true - helixRepo: dotnet/fsharp - jobs: - - - job: WindowsLangVersionPreview - pool: - # The PR build definition sets this variable: - # WindowsMachineQueueName=Windows.vs2022.amd64.open - # and there is an alternate build definition that sets this to a queue that is always scouting the - # next preview of Visual Studio. - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - timeoutInMinutes: 120 - steps: - - checkout: self - clean: true - - - script: eng\CIBuild.cmd -compressallmetadata -configuration Release /p:FSharpLangVersion=preview - env: - NativeToolsOnMachine: true - displayName: Build - - - task: PublishBuildArtifacts@1 - displayName: Publish Build BinLog - condition: always() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows Release build binlogs' - ArtifactType: Container - parallel: true - - - job: WindowsNoRealsig_testCoreclr - pool: - # The PR build definition sets this variable: - # WindowsMachineQueueName=Windows.vs2022.amd64.open - # and there is an alternate build definition that sets this to a queue that is always scouting the - # next preview of Visual Studio. - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - timeoutInMinutes: 120 - steps: - - checkout: self - clean: true - - - script: eng\CIBuild.cmd -compressallmetadata -norealsig -testCoreclr -configuration Release - env: - NativeToolsOnMachine: true - displayName: Build - - - task: PublishBuildArtifacts@1 - displayName: Publish Build BinLog - condition: always() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows Release build binlogs' - ArtifactType: Container - parallel: true - - - job: WindowsNoRealsig_testDesktop - pool: - # The PR build definition sets this variable: - # WindowsMachineQueueName=Windows.vs2022.amd64.open - # and there is an alternate build definition that sets this to a queue that is always scouting the - # next preview of Visual Studio. - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - timeoutInMinutes: 120 - steps: - - checkout: self - clean: true - - - script: eng\CIBuild.cmd -compressallmetadata -norealsig -testDesktop -configuration Release - env: - NativeToolsOnMachine: true - displayName: Build - - - task: PublishBuildArtifacts@1 - displayName: Publish Build BinLog - condition: always() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows Release build binlogs' - ArtifactType: Container - parallel: true - - - job: WindowsStrictIndentation - pool: - # The PR build definition sets this variable: - # WindowsMachineQueueName=Windows.vs2022.amd64.open - # and there is an alternate build definition that sets this to a queue that is always scouting the - # next preview of Visual Studio. - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - timeoutInMinutes: 120 - steps: - - checkout: self - clean: true - - - script: eng\CIBuild.cmd -compressallmetadata -configuration Release /p:AdditionalFscCmdFlags=--strict-indentation+ - env: - NativeToolsOnMachine: true - displayName: Build - - - task: PublishBuildArtifacts@1 - displayName: Publish Build BinLog - condition: always() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows Release build binlogs' - ArtifactType: Container - parallel: true - - - job: WindowsNoStrictIndentation - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - timeoutInMinutes: 120 - steps: - - checkout: self - clean: true - - - script: eng\CIBuild.cmd -compressallmetadata -configuration Release /p:AdditionalFscCmdFlags=--strict-indentation- - env: - NativeToolsOnMachine: true - displayName: Build - - - task: PublishBuildArtifacts@1 - displayName: Publish Build BinLog - condition: always() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows Release build binlogs' - ArtifactType: Container - parallel: true - - # Windows With Compressed Metadata - - job: WindowsCompressedMetadata - variables: - - name: XUNIT_LOGS - value: $(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration) - - name: __VSNeverShowWhatsNew - value: 1 - pool: - # The PR build definition sets this variable: - # WindowsMachineQueueName=Windows.vs2022.amd64.open - # and there is an alternate build definition that sets this to a queue that is always scouting the - # next preview of Visual Studio. - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - timeoutInMinutes: 120 - strategy: - maxParallel: 5 - matrix: - desktop_release: - _configuration: Release - _testKind: testDesktop - coreclr_release: - _configuration: Release - _testKind: testCoreclr - fsharpqa_release: - _configuration: Release - _testKind: testFSharpQA - vs_release: - _configuration: Release - _testKind: testVs - transparent_compiler_release: - _configuration: Release - _testKind: testCoreclr - - ${{ if eq(variables['Build.Reason'], 'Flaky, disabled, was PullRequest') }}: - inttests_release: - _configuration: Release - _testKind: testIntegration - steps: - - checkout: self - clean: true - - - powershell: eng\SetupVSHive.ps1 - displayName: Setup VS Hive - condition: or(eq(variables['_testKind'], 'testVs'), eq(variables['_testKind'], 'testIntegration')) - - # yes, this is miserable, but - https://github.com/dotnet/arcade/issues/13239 - - script: eng\CIBuild.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) - env: - NativeToolsOnMachine: true - displayName: Build / Test - condition: and( ne(variables['_testKind'], 'testIntegration'), ne(variables['System.JobName'], 'transparent_compiler_release') ) - - - script: eng\CIBuild.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) - env: - TEST_TRANSPARENT_COMPILER: 1 - NativeToolsOnMachine: true - displayName: Build / Test Transparent Compiler - condition: and( eq(variables['System.JobName'], 'transparent_compiler_release'), ne(variables['_testKind'], 'testIntegration') ) - - - script: eng\CIBuild.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) - env: - NativeToolsOnMachine: true - displayName: Build / Integration Test - continueOnError: true - condition: eq(variables['_testKind'], 'testIntegration') - - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'NUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)' - continueOnError: true - condition: ne(variables['_testKind'], 'testFSharpQA') - - task: PublishBuildArtifacts@1 - displayName: Publish Tests BinLog - condition: always() - continueOnError: true - inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/$(_configuration)\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows $(_configuration) $(_testKind) test binlogs' - ArtifactType: Container - parallel: true - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration)' - ArtifactName: 'Windows $(_configuration) $(_testKind) test logs' - publishLocation: Container - continueOnError: true - condition: always() - - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj - displayName: Dump NuGet cache contents - condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish NuGet cache contents - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows $(_testKind)' - publishLocation: Container - continueOnError: true - condition: failed() - - # Mock official build - - job: MockOfficial - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - steps: - - checkout: self - clean: true - - pwsh: .\eng\MockBuild.ps1 - displayName: Build with OfficialBuildId - - # Linux - - job: Linux - pool: - vmImage: $(UbuntuMachineQueueName) - variables: - - name: _SignType - value: Test - steps: - - checkout: self - clean: true - - script: ./eng/cibuild.sh --configuration $(_BuildConfig) --testcoreclr - displayName: Build / Test - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'NUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - continueOnError: true - condition: always() - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - ArtifactName: 'Linux $(_BuildConfig) test logs' - publishLocation: Container - continueOnError: true - condition: failed() - - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj - displayName: Dump NuGet cache contents - condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish NuGet cache contents - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Linux' - publishLocation: Container - continueOnError: true - condition: failed() - - # MacOS - - job: MacOS - pool: - vmImage: macos-11 - variables: - - name: _SignType - value: Test - steps: - - checkout: self - clean: true - - script: ./eng/cibuild.sh --configuration $(_BuildConfig) --testcoreclr - env: - COMPlus_DefaultStackSize: 1000000 - displayName: Build / Test - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'NUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - continueOnError: true - condition: always() - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - ArtifactName: 'MacOS $(_BuildConfig) test logs' - publishLocation: Container - continueOnError: true - condition: failed() - - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj - displayName: Dump NuGet cache contents - condition: failed() - - task: PublishBuildArtifacts@1 - displayName: Publish NuGet cache contents - inputs: - PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Mac' - publishLocation: Container - continueOnError: true - condition: failed() - - # End to end build - - job: EndToEndBuildTests - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - strategy: - maxParallel: 2 - matrix: - regular: - _experimental_flag: '' - experimental_features: - _experimental_flag: '' - steps: - - checkout: self - clean: true - - script: .\Build.cmd -c Release -pack - env: - NativeToolsOnMachine: true - FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) - - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c Release - env: - FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) - displayName: End to end build tests - - # Up-to-date - disabled due to it being flaky - #- job: UpToDate_Windows - # pool: - # vmImage: windows-latest - # steps: - # - checkout: self - # clean: true - # - task: PowerShell@2 - # displayName: Run up-to-date build check - # inputs: - # filePath: eng\tests\UpToDate.ps1 - # arguments: -configuration $(_BuildConfig) -ci -binaryLog - - # Run Build with Fsharp Experimental Features - # Possible change: --times:$(Build.SourcesDirectory)/artifacts/log/Release/compiler_timing.csv - - # Plain FCS build Windows - - job: Plain_Build_Windows - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - variables: - - name: _BuildConfig - value: Debug - steps: - - checkout: self - clean: true - - script: dotnet --list-sdks - displayName: Report dotnet SDK versions - - task: UseDotNet@2 - displayName: install SDK - inputs: - packageType: sdk - useGlobalJson: true - includePreviewVersions: true - workingDirectory: $(Build.SourcesDirectory) - installationPath: $(Agent.ToolsDirectory)/dotnet - - script: dotnet build .\FSharp.Compiler.Service.sln /bl:\"artifacts/log/$(_BuildConfig)/ServiceRegularBuild.binlog\" - workingDirectory: $(Build.SourcesDirectory) - displayName: Regular rebuild of FSharp.Compiler.Service.sln - continueOnError: true - condition: always() - - # Plain FCS build Linux - - job: Plain_Build_Linux - pool: - vmImage: $(UbuntuMachineQueueName) - variables: - - name: _BuildConfig - value: Debug - steps: - - checkout: self - clean: true - - script: dotnet --list-sdks - displayName: Report dotnet SDK versions - - task: UseDotNet@2 - displayName: install SDK - inputs: - packageType: sdk - useGlobalJson: true - includePreviewVersions: true - workingDirectory: $(Build.SourcesDirectory) - installationPath: $(Agent.ToolsDirectory)/dotnet - - script: dotnet build ./FSharp.Compiler.Service.sln /bl:\"artifacts/log/$(_BuildConfig)/ServiceRegularBuild.binlog\" - workingDirectory: $(Build.SourcesDirectory) - displayName: Regular rebuild of FSharp.Compiler.Service.sln - continueOnError: true - condition: always() - - # Plain FCS build Mac - - job: Plain_Build_MacOS - pool: - vmImage: macos-11 - variables: - - name: _BuildConfig - value: Debug - steps: - - checkout: self - clean: true - - script: dotnet --list-sdks - displayName: Report dotnet SDK versions - - task: UseDotNet@2 - displayName: install SDK - inputs: - packageType: sdk - useGlobalJson: true - includePreviewVersions: true - workingDirectory: $(Build.SourcesDirectory) - installationPath: $(Agent.ToolsDirectory)/dotnet - - script: dotnet build ./FSharp.Compiler.Service.sln /bl:\"artifacts/log/$(_BuildConfig)/ServiceRegularBuild.binlog\" - workingDirectory: $(Build.SourcesDirectory) - displayName: Regular rebuild of FSharp.Compiler.Service.sln - continueOnError: true - condition: always() - - # Build and run fast benchmarks - - job: Benchmarks - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - variables: - - name: _BuildConfig - value: Release - steps: - - checkout: self - clean: true - - script: eng\CIBuild.cmd -configuration $(_BuildConfig) -testBenchmarks - displayName: Smoke test fast benchmarks - continueOnError: true - condition: always() - - # Test trimming on Windows - - job: Build_And_Test_AOT_Windows - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - strategy: - maxParallel: 2 - matrix: - compressed_metadata: - _kind: "-compressAllMetadata" - classic_metadata: - _kind: "" - variables: - - name: _BuildConfig - value: Release - steps: - - checkout: self - clean: true - - task: UseDotNet@2 - displayName: install SDK - inputs: - packageType: sdk - useGlobalJson: true - includePreviewVersions: true - workingDirectory: $(Build.SourcesDirectory) - installationPath: $(Agent.ToolsDirectory)/dotnet - - script: dotnet --list-sdks - displayName: Report dotnet SDK versions - - script: .\Build.cmd $(_kind) -pack -c $(_BuildConfig) - env: - NativeToolsOnMachine: true - displayName: Initial build and prepare packages. - - script: $(Build.SourcesDirectory)/tests/AheadOfTime/check.cmd - displayName: Build, trim, publish and check the state of the trimmed app. - workingDirectory: $(Build.SourcesDirectory)/tests/AheadOfTime - - task: PublishPipelineArtifact@1 - displayName: Publish Trim Tests Logs - inputs: - targetPath: './artifacts/log/Release/AheadOfTime/Trimming/' - artifactName: 'Trim Test Logs Attempt $(System.JobAttempt) Logs $(_kind)' - continueOnError: true - condition: always() - - # Arcade-powered source build - # turned off until https://github.com/dotnet/source-build/issues/1795 is fixed - # - template: /eng/common/templates/jobs/jobs.yml - # parameters: - # enablePublishUsingPipelines: true - # enablePublishBuildArtifacts: true - # enablePublishBuildAssets: true - # artifacts: - # publish: - # artifacts: true - # manifests: true - # runSourceBuild: true - # sourceBuildParameters: - # includeDefaultManagedPlatform: true - -#---------------------------------------------------------------------------------------------------------------------# -# Post Build # -#---------------------------------------------------------------------------------------------------------------------# -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: eng/common/templates/post-build/post-build.yml - parameters: - publishingInfraVersion: 3 - # Symbol validation is not entirely reliable as of yet, so should be turned off until https://github.com/dotnet/arcade/issues/2871 is resolved. - enableSymbolValidation: false - # SourceLink improperly looks for generated files. See https://github.com/dotnet/arcade/issues/3069 - enableSourceLinkValidation: false - # Enable SDL validation, passing through values from the 'DotNet-FSharp-SDLValidation-Params' group. - SDLValidationParameters: - enable: true - params: >- - -SourceToolsList @("policheck","credscan") - -ArtifactToolsList @("binskim") - -BinskimAdditionalRunConfigParams @("IgnorePdbLoadError < True","Recurse < True") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "FSharp" - -TsaCodebaseName "FSharp-GitHub" - -TsaPublish $True - -PoliCheckAdditionalRunConfigParams @("UserExclusionPath < $(Build.SourcesDirectory)/eng/policheck_exclusions.xml") - -#---------------------------------------------------------------------------------------------------------------------# -# VS Insertion # -#---------------------------------------------------------------------------------------------------------------------# -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: eng/release/insert-into-vs.yml - parameters: - componentBranchName: refs/heads/release/dev17.9 - insertTargetBranch: rel/d17.9 - insertTeamEmail: fsharpteam@microsoft.com - insertTeamName: 'F#' - completeInsertion: 'auto' + componentBranchName: refs/heads/main + insertTargetBranch: main + insertTeamEmail: fsharpteam@microsoft.com + insertTeamName: 'F#' + completeInsertion: 'auto' From bcd2fdf4c32ad23082ce2b6ec12396738cd1410d Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 12:54:38 +0100 Subject: [PATCH 02/34] 1ES pipelines migration --- azure-pipelines.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 601c1ebb99..960daab7dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -25,9 +25,9 @@ trigger: - TESTGUIDE.md variables: - - name: ReleaseBranch - value: release/dev17.9 - - name: VSTargetInsertionBranch + - name: FSharpReleaseBranchName + value: 1espm + - name: VSInsertionTargetBranchName value: main - name: _TeamName value: FSharp @@ -74,11 +74,12 @@ extends: #-------------------------------------------------------------------------------------------------------------------# # Signed build # #-------------------------------------------------------------------------------------------------------------------# + # Localization: we only run it for specific release branches. - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/release/dev17.9') }}: - template: /eng/common/templates-official/job/onelocbuild.yml@self parameters: MirrorRepo: fsharp - MirrorBranch: release/dev17.9 + MirrorBranch: $(FSharpReleaseBranchName) LclSource: lclFilesfromPackage LclPackageId: 'LCL-JUNO-PROD-FSHARP' - template: /eng/common/templates-official/jobs/jobs.yml@self @@ -213,8 +214,8 @@ extends: #---------------------------------------------------------------------------------------------------------------------# - template: eng/release/insert-into-vs.yml parameters: - componentBranchName: refs/heads/main - insertTargetBranch: main + componentBranchName: refs/heads/$(FSharpReleaseBranchName) + insertTargetBranch: $(VSInsertionBranchName) insertTeamEmail: fsharpteam@microsoft.com insertTeamName: 'F#' completeInsertion: 'auto' From bd5914f2880df164268bcbb9675ecc980fda78c8 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 13:29:21 +0100 Subject: [PATCH 03/34] 1ES pipelines migration, please work? --- azure-pipelines.yml | 83 +++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 48 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 960daab7dd..f6cc40da34 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -99,6 +99,41 @@ extends: demands: 1es-windows-2022-pt os: windows timeoutInMinutes: 300 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish Test Logs' + condition: and(succeededOrFailed(), ne(variables['SkipTests'], 'true')) + targetPath: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig)' + artifactName: 'Test Logs' + - output: pipelineArtifact + displayName: 'Publish Artifact Packages' + condition: succeeded() + targetPath: '$(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig)' + artifactName: 'Packages' + - output: pipelineArtifact + displayName: 'Publish Artifact VSSetup' + condition: succeeded() + targetPath: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' + artifactName: 'VSSetup' + - output: pipelineArtifact + displayName: 'Publish Nightly VSIX' + condition: succeeded() + targetPath: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpDebug.vsix' + artifactName: 'Nightly' + - output: pipelineArtifact + displayName: 'Publish Artifact Symbols' + condition: succeeded() + targetPath: '$(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig)' + artifactName: 'NativeSymbols' + - output: microBuildVstsDrop + dropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' + dropName: $(VisualStudioDropName) + dropRetentionDays: 90 + accessToken: $(dn-bot-devdiv-drop-rw-code-rw) + dropServiceUri: 'https://devdiv.artifacts.visualstudio.com' + vsDropServiceUri: 'https://vsdrop.corp.microsoft.com/file/v1' + variables: - group: DotNet-Symbol-Server-Pats - group: DotNet-DevDiv-Insertion-Workflow-Variables @@ -132,54 +167,6 @@ extends: displayName: End to end build tests continueOnError: true condition: always() - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'NUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - continueOnError: true - condition: ne(variables['SkipTests'], 'true') - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig)' - ArtifactName: 'Test Logs' - publishLocation: Container - continueOnError: true - condition: ne(variables['SkipTests'], 'true') - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact Packages - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig)' - ArtifactName: 'Packages' - condition: succeeded() - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact VSSetup - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' - ArtifactName: 'VSSetup' - condition: succeeded() - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact Nightly - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpDebug.vsix' - ArtifactName: 'Nightly' - condition: succeeded() - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact Symbols - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig)' - ArtifactName: 'NativeSymbols' - condition: succeeded() - - task: ms-vseng.MicroBuildTasks.4305a8de-ba66-4d8b-b2d1-0dc4ecbbf5e8.MicroBuildUploadVstsDropFolder@1 - displayName: Upload VSTS Drop - inputs: - DropName: $(VisualStudioDropName) - DropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' - AccessToken: $(dn-bot-devdiv-drop-rw-code-rw) - condition: succeeded() - #---------------------------------------------------------------------------------------------------------------------# # Post Build # From feb9be6599f26537b90a499219524fdae77226c5 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 13:35:05 +0100 Subject: [PATCH 04/34] 1ES pipelines migration, please work? --- azure-pipelines.yml | 10 +++++++++- eng/release/insert-into-vs.yml | 7 ++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f6cc40da34..b52ea018d2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -167,11 +167,19 @@ extends: displayName: End to end build tests continueOnError: true condition: always() + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'NUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + continueOnError: true + condition: ne(variables['SkipTests'], 'true') #---------------------------------------------------------------------------------------------------------------------# # Post Build # #---------------------------------------------------------------------------------------------------------------------# - - template: eng/common/templates/post-build/post-build.yml + - template: eng/common/templates-official/post-build/post-build.yml parameters: publishingInfraVersion: 3 # Symbol validation is not entirely reliable as of yet, so should be turned off until https://github.com/dotnet/arcade/issues/2871 is resolved. diff --git a/eng/release/insert-into-vs.yml b/eng/release/insert-into-vs.yml index 63973516ba..49d4e133d3 100644 --- a/eng/release/insert-into-vs.yml +++ b/eng/release/insert-into-vs.yml @@ -14,8 +14,9 @@ stages: jobs: - job: Insert_VS pool: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals windows.vs2019.amd64 + name: $(DncEngInternalBuildPool) + demands: 1es-windows-2022-pt + os: windows variables: - group: DotNet-VSTS-Infra-Access - name: InsertAccessToken @@ -69,5 +70,5 @@ stages: - task: ms-vseng.MicroBuildShipTasks.55100717-a81d-45ea-a363-b8fe3ec375ad.MicroBuildInsertVsPayload@3 displayName: 'Insert VS Payload' inputs: - LinkWorkItemsToPR: false + LinkWorkItemsToPR: true condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], '${{ parameters.componentBranchName }}'), eq(variables['Build.SourceBranch'], 'refs/heads/${{ parameters.componentBranchName }}'))) From 5a24009caefc83f32b38d45e98643ba4c767dba3 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 14:30:55 +0100 Subject: [PATCH 05/34] 1ES pipelines migration, please work? --- azure-pipelines.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b52ea018d2..92c7a07176 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -60,13 +60,15 @@ resources: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: + featureFlags: + autoBaseline: true sdl: + sbom: + enabled: false # VS SBOM is generated with other steps sourceAnalysisPool: name: $(DncEngInternalBuildPool) image: 1es-windows-2022-pt os: windows - customBuildTags: - - ES365AIMigrationTooling stages: - stage: build displayName: Build @@ -100,6 +102,12 @@ extends: os: windows timeoutInMinutes: 300 templateContext: + mb: + signing: + enabled: true + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + signType: $(_SignType) + zipSources: false outputs: - output: pipelineArtifact displayName: 'Publish Test Logs' From 285280ab5cafb29718cdceac436d647953e13010 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 14:35:42 +0100 Subject: [PATCH 06/34] 1ES pipelines migration, please work? --- azure-pipelines.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 92c7a07176..6ed2daf4b4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -60,8 +60,8 @@ resources: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: - featureFlags: - autoBaseline: true + #featureFlags: + # autoBaseline: true sdl: sbom: enabled: false # VS SBOM is generated with other steps @@ -171,18 +171,6 @@ extends: /p:GenerateSbom=true env: NativeToolsOnMachine: true - - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c $(_BuildConfig) - displayName: End to end build tests - continueOnError: true - condition: always() - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'NUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - continueOnError: true - condition: ne(variables['SkipTests'], 'true') #---------------------------------------------------------------------------------------------------------------------# # Post Build # From 65c93d822efda16d22e043fe47b624762ddfb122 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 14:43:36 +0100 Subject: [PATCH 07/34] 1ES pipelines migration, please work? --- azure-pipelines.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6ed2daf4b4..120bf09a4c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -63,8 +63,13 @@ extends: #featureFlags: # autoBaseline: true sdl: + binskim: + enabled: true + policheck: + enabled: true sbom: enabled: false # VS SBOM is generated with other steps + justificationForDisabling: 'SBOM for F# is generated via build process.' sourceAnalysisPool: name: $(DncEngInternalBuildPool) image: 1es-windows-2022-pt @@ -112,27 +117,27 @@ extends: - output: pipelineArtifact displayName: 'Publish Test Logs' condition: and(succeededOrFailed(), ne(variables['SkipTests'], 'true')) - targetPath: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig)' + targetPath: $(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig) artifactName: 'Test Logs' - output: pipelineArtifact displayName: 'Publish Artifact Packages' condition: succeeded() - targetPath: '$(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig)' + targetPath: $(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig) artifactName: 'Packages' - output: pipelineArtifact displayName: 'Publish Artifact VSSetup' condition: succeeded() - targetPath: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' + targetPath: $(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion artifactName: 'VSSetup' - output: pipelineArtifact displayName: 'Publish Nightly VSIX' condition: succeeded() - targetPath: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpDebug.vsix' + targetPath: $(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpDebug.vsix artifactName: 'Nightly' - output: pipelineArtifact displayName: 'Publish Artifact Symbols' condition: succeeded() - targetPath: '$(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig)' + targetPath: $(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig) artifactName: 'NativeSymbols' - output: microBuildVstsDrop dropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' From 13a64a56011880d88b4b4c555fc8bb80a7671e16 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 14:58:55 +0100 Subject: [PATCH 08/34] 1ES pipelines migration, please work? --- azure-pipelines.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 120bf09a4c..ef970c8fd4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -99,10 +99,13 @@ extends: enableSourceBuild: true enableTelemetry: true helixRepo: dotnet/fsharp + poolName: $(DncEngInternalBuildPool) + poolDemands: 1es-windows-2022-pt + poolOs: windows jobs: - job: Full_Signed pool: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Svc-Internal demands: 1es-windows-2022-pt os: windows timeoutInMinutes: 300 From 609753a2548d62feb554c0efe9deff3b8646e314 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 15:13:45 +0100 Subject: [PATCH 09/34] Temporary fix Arcade's nuget script --- eng/common/SetupNugetSources.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 6c65e81925..6a05966d57 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -25,7 +25,7 @@ [CmdletBinding()] param ( [Parameter(Mandatory = $true)][string]$ConfigFile, - [Parameter(Mandatory = $true)][string]$Password + [Parameter(Mandatory = $true)][SecureString]$Password ) $ErrorActionPreference = "Stop" @@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $Password) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, [SecureString] $creds, $Username, [SecureString] $Password) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -53,7 +53,7 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Usern } # Add a credential node for the specified source -function AddCredential($creds, $source, $username, $password) { +function AddCredential([SecureString] $creds, $source, $username, [SecureString] $password) { # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -85,7 +85,7 @@ function AddCredential($creds, $source, $username, $password) { $passwordElement.SetAttribute("value", $Password) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, [SecureString] $Creds, $Username, [SecureString] $Password) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." From 6259ff135a66b919d78a6f492425fafa8d699fd9 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 15:18:08 +0100 Subject: [PATCH 10/34] Fix pools and images parameters --- eng/release/insert-into-vs.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/eng/release/insert-into-vs.yml b/eng/release/insert-into-vs.yml index 49d4e133d3..ad5b7b49fd 100644 --- a/eng/release/insert-into-vs.yml +++ b/eng/release/insert-into-vs.yml @@ -14,9 +14,8 @@ stages: jobs: - job: Insert_VS pool: - name: $(DncEngInternalBuildPool) - demands: 1es-windows-2022-pt - os: windows + name: NetCore1ESPool-Svc-Internal + image: 1es-windows-2022-pt variables: - group: DotNet-VSTS-Infra-Access - name: InsertAccessToken From c51804a9b2feab4765f0a1a050ff4e5a3c1984a4 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 15:19:41 +0100 Subject: [PATCH 11/34] Use vs2022 preview pool: --- azure-pipelines.yml | 6 +----- eng/release/insert-into-vs.yml | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ef970c8fd4..5e01f066d6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -99,15 +99,11 @@ extends: enableSourceBuild: true enableTelemetry: true helixRepo: dotnet/fsharp - poolName: $(DncEngInternalBuildPool) - poolDemands: 1es-windows-2022-pt - poolOs: windows jobs: - job: Full_Signed pool: name: NetCore1ESPool-Svc-Internal - demands: 1es-windows-2022-pt - os: windows + image: windows.vs2022preview.amd64 timeoutInMinutes: 300 templateContext: mb: diff --git a/eng/release/insert-into-vs.yml b/eng/release/insert-into-vs.yml index ad5b7b49fd..746b73d7c6 100644 --- a/eng/release/insert-into-vs.yml +++ b/eng/release/insert-into-vs.yml @@ -15,7 +15,7 @@ stages: - job: Insert_VS pool: name: NetCore1ESPool-Svc-Internal - image: 1es-windows-2022-pt + image: windows.vs2022preview.amd64 variables: - group: DotNet-VSTS-Infra-Access - name: InsertAccessToken From 9bf82de444057687e92af5bd2a6a0a5c1dee3ae2 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 16:36:20 +0100 Subject: [PATCH 12/34] Add PR pipelines (for now just copy of original one): --- azure-pipelines-PR.yml | 891 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 891 insertions(+) create mode 100644 azure-pipelines-PR.yml diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml new file mode 100644 index 0000000000..a680b0b557 --- /dev/null +++ b/azure-pipelines-PR.yml @@ -0,0 +1,891 @@ +# CI and PR triggers +trigger: + branches: + include: + - main + - feature/* + - release/* + paths: + include: + - '*' + exclude: + - .github/* + - docs/ + - .vscode/* + - .devcontainer/* + - tests/scripts/ + - attributions.md + - CODE_OF_CONDUCT.md + - DEVGUIDE.md + - INTERNAL.md + - Language-Version-History.md + - License.txt + - README.md + - release-notes.md + - TESTGUIDE.md + +pr: + branches: + include: + - main + - feature/* + - release/* + paths: + include: + - '*' + exclude: + - .github/* + - docs/ + - attributions.md + - CODE_OF_CONDUCT.md + - DEVGUIDE.md + - INTERNAL.md + - Language-Version-History.md + - License.txt + - README.md + - release-notes.md + - TESTGUIDE.md + +variables: + - name: _TeamName + value: FSharp + - name: _BuildConfig + value: Release + - name: _PublishUsingPipelines + value: true + - name: _DotNetArtifactsCategory + value: .NETCore + - name: VisualStudioDropName + value: Products/$(System.TeamProject)/$(Build.Repository.Name)/$(Build.SourceBranchName)/$(Build.BuildNumber) + - name: Codeql.Enabled + value: true + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - name: _DotNetValidationArtifactsCategory + value: .NETCoreValidation + - group: DotNet-FSharp-SDLValidation-Params + - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: + - name: RunningAsPullRequest + value: true + # Pick up pool provider name behavior from shared yaml template + - template: /eng/common/templates/variables/pool-providers.yml + +# Variables defined in yml cannot be overridden at queue time; instead overridable variables must be defined in the web UI. +# Commenting out until something like this is supported: https://github.com/Microsoft/azure-pipelines-yaml/pull/129 +#variables: +#- name: SkipTests +# defaultValue: false + +stages: +- stage: build + displayName: Build + jobs: + + #-------------------------------------------------------------------------------------------------------------------# + # Signed build # + #-------------------------------------------------------------------------------------------------------------------# + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/release/dev17.9') }}: + - template: /eng/common/templates/job/onelocbuild.yml + parameters: + MirrorRepo: fsharp + MirrorBranch: release/dev17.9 + LclSource: lclFilesfromPackage + LclPackageId: 'LCL-JUNO-PROD-FSHARP' + - template: /eng/common/templates/jobs/jobs.yml + parameters: + enableMicrobuild: true + enablePublishBuildArtifacts: true + enablePublishTestResults: false + enablePublishBuildAssets: true + enablePublishUsingPipelines: $(_PublishUsingPipelines) + enableSourceBuild: true + enableTelemetry: true + helixRepo: dotnet/fsharp + jobs: + - job: Full_Signed + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2022preview.amd64 + timeoutInMinutes: 300 + variables: + - group: DotNet-Symbol-Server-Pats + - group: DotNet-DevDiv-Insertion-Workflow-Variables + - name: _SignType + value: Real + steps: + - checkout: self + clean: true + - template: /eng/restore-internal-tools.yml + - script: eng\CIBuild.cmd + -configuration $(_BuildConfig) + -prepareMachine + -testAllButIntegration + -officialSkipTests $(SkipTests) + /p:SignType=$(_SignType) + /p:DotNetSignType=$(_SignType) + /p:MicroBuild_SigningEnabled=true + /p:OverridePackageSource=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + /p:PublishToSymbolServer=true + /p:VisualStudioDropName=$(VisualStudioDropName) + /p:GenerateSbom=true + env: + NativeToolsOnMachine: true + - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c $(_BuildConfig) + displayName: End to end build tests + continueOnError: true + condition: always() + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'NUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + continueOnError: true + condition: ne(variables['SkipTests'], 'true') + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig)' + ArtifactName: 'Test Logs' + publishLocation: Container + continueOnError: true + condition: ne(variables['SkipTests'], 'true') + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact Packages + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig)' + ArtifactName: 'Packages' + condition: succeeded() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact VSSetup + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' + ArtifactName: 'VSSetup' + condition: succeeded() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact Nightly + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpDebug.vsix' + ArtifactName: 'Nightly' + condition: succeeded() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact Symbols + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig)' + ArtifactName: 'NativeSymbols' + condition: succeeded() + - task: ms-vseng.MicroBuildTasks.4305a8de-ba66-4d8b-b2d1-0dc4ecbbf5e8.MicroBuildUploadVstsDropFolder@1 + displayName: Upload VSTS Drop + inputs: + DropName: $(VisualStudioDropName) + DropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' + AccessToken: $(dn-bot-devdiv-drop-rw-code-rw) + condition: succeeded() + + #-------------------------------------------------------------------------------------------------------------------# + # PR builds without logs publishing # + #-------------------------------------------------------------------------------------------------------------------# + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/jobs/jobs.yml + parameters: + enableMicrobuild: false + enablePublishBuildArtifacts: false + enablePublishTestResults: false + enablePublishBuildAssets: false + enablePublishUsingPipelines: $(_PublishUsingPipelines) + enableSourceBuild: false + enableTelemetry: true + helixRepo: dotnet/fsharp + jobs: + # Determinism, we want to run it only in PR builds + - job: Determinism_Debug + condition: eq(variables['Build.Reason'], 'PullRequest') + variables: + - name: _SignType + value: Test + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 90 + strategy: + maxParallel: 2 + matrix: + regular: + _experimental_flag: '' + experimental_features: + _experimental_flag: '' + steps: + - checkout: self + clean: true + - task: UseDotNet@2 + displayName: install SDK + inputs: + packageType: sdk + useGlobalJson: true + includePreviewVersions: false + workingDirectory: $(Build.SourcesDirectory) + installationPath: $(Build.SourcesDirectory)/.dotnet + - script: .\eng\test-determinism.cmd -configuration Debug + env: + FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) + displayName: Determinism tests with Debug configuration + - task: PublishPipelineArtifact@1 + displayName: Publish Determinism Logs + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts/log/Debug' + artifactName: 'Determinism_Debug Attempt $(System.JobAttempt) Logs' + continueOnError: true + condition: not(succeeded()) + + # Check FSComp.txt error code sorting and code formatting + - job: CheckCodeFormatting + pool: + vmImage: $(UbuntuMachineQueueName) + steps: + - checkout: self + clean: true + - script: dotnet --list-sdks + displayName: Report dotnet SDK versions + - task: UseDotNet@2 + displayName: install SDK + inputs: + packageType: sdk + useGlobalJson: true + includePreviewVersions: true + workingDirectory: $(Build.SourcesDirectory) + installationPath: $(Agent.ToolsDirectory)/dotnet + - script: dotnet tool restore + env: + DOTNET_ROLL_FORWARD_TO_PRERELEASE: 1 + displayName: Install tools + - script: dotnet fsi src/Compiler/FSCompCheck.fsx + env: + DOTNET_ROLL_FORWARD_TO_PRERELEASE: 1 + displayName: Check error code sorting in src/Compiler/FSComp.txt + - script: dotnet fantomas . --check + env: + DOTNET_ROLL_FORWARD_TO_PRERELEASE: 1 + displayName: Check code formatting (run 'dotnet fantomas .' to fix) + + # Check whether package with current version has been published to nuget.org + # We will try to restore both FSharp.Core and FCS and if restore is _successful_, package version needs to be bumped. + # NOTE: The check now runs on all branches (not just release), + # because we want to catch cases when version is desynched and we didn't update it. + # It is also helping the release notes automation to be up to date with packages versions. + - job: Check_Published_Package_Versions + # condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), or(startsWith(variables['System.PullRequest.SourceBranch'], 'release/dev'), startsWith(variables['System.PullRequest.TargetBranch'], 'release/dev'))) + pool: + vmImage: $(UbuntuMachineQueueName) + strategy: + maxParallel: 2 + matrix: + FCS: + _project: "FSharp.Compiler.Service_notshipped.fsproj" + FSCore: + _project: "FSharp.Core_notshipped.fsproj" + steps: + - checkout: self + clean: true + - task: UseDotNet@2 + displayName: install SDK + inputs: + packageType: sdk + useGlobalJson: true + includePreviewVersions: true + workingDirectory: $(Build.SourcesDirectory) + installationPath: $(Agent.ToolsDirectory)/dotnet + - pwsh: ./check.ps1 -project $(_project) + workingDirectory: $(Build.SourcesDirectory)/buildtools/checkpackages + env: + DOTNET_ROLL_FORWARD_TO_PRERELEASE: 1 + displayName: Check published package version + + + #-------------------------------------------------------------------------------------------------------------------# + # PR builds # + #-------------------------------------------------------------------------------------------------------------------# + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/jobs/jobs.yml + parameters: + enableMicrobuild: true + enablePublishBuildArtifacts: true + enablePublishTestResults: false + enablePublishBuildAssets: true + enablePublishUsingPipelines: $(_PublishUsingPipelines) + enableSourceBuild: true + enableTelemetry: true + helixRepo: dotnet/fsharp + jobs: + + - job: WindowsLangVersionPreview + pool: + # The PR build definition sets this variable: + # WindowsMachineQueueName=Windows.vs2022.amd64.open + # and there is an alternate build definition that sets this to a queue that is always scouting the + # next preview of Visual Studio. + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 120 + steps: + - checkout: self + clean: true + + - script: eng\CIBuild.cmd -compressallmetadata -configuration Release /p:FSharpLangVersion=preview + env: + NativeToolsOnMachine: true + displayName: Build + + - task: PublishBuildArtifacts@1 + displayName: Publish Build BinLog + condition: always() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + ArtifactName: 'Windows Release build binlogs' + ArtifactType: Container + parallel: true + + - job: WindowsNoRealsig_testCoreclr + pool: + # The PR build definition sets this variable: + # WindowsMachineQueueName=Windows.vs2022.amd64.open + # and there is an alternate build definition that sets this to a queue that is always scouting the + # next preview of Visual Studio. + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 120 + steps: + - checkout: self + clean: true + + - script: eng\CIBuild.cmd -compressallmetadata -norealsig -testCoreclr -configuration Release + env: + NativeToolsOnMachine: true + displayName: Build + + - task: PublishBuildArtifacts@1 + displayName: Publish Build BinLog + condition: always() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + ArtifactName: 'Windows Release build binlogs' + ArtifactType: Container + parallel: true + + - job: WindowsNoRealsig_testDesktop + pool: + # The PR build definition sets this variable: + # WindowsMachineQueueName=Windows.vs2022.amd64.open + # and there is an alternate build definition that sets this to a queue that is always scouting the + # next preview of Visual Studio. + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 120 + steps: + - checkout: self + clean: true + + - script: eng\CIBuild.cmd -compressallmetadata -norealsig -testDesktop -configuration Release + env: + NativeToolsOnMachine: true + displayName: Build + + - task: PublishBuildArtifacts@1 + displayName: Publish Build BinLog + condition: always() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + ArtifactName: 'Windows Release build binlogs' + ArtifactType: Container + parallel: true + + - job: WindowsStrictIndentation + pool: + # The PR build definition sets this variable: + # WindowsMachineQueueName=Windows.vs2022.amd64.open + # and there is an alternate build definition that sets this to a queue that is always scouting the + # next preview of Visual Studio. + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 120 + steps: + - checkout: self + clean: true + + - script: eng\CIBuild.cmd -compressallmetadata -configuration Release /p:AdditionalFscCmdFlags=--strict-indentation+ + env: + NativeToolsOnMachine: true + displayName: Build + + - task: PublishBuildArtifacts@1 + displayName: Publish Build BinLog + condition: always() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + ArtifactName: 'Windows Release build binlogs' + ArtifactType: Container + parallel: true + + - job: WindowsNoStrictIndentation + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 120 + steps: + - checkout: self + clean: true + + - script: eng\CIBuild.cmd -compressallmetadata -configuration Release /p:AdditionalFscCmdFlags=--strict-indentation- + env: + NativeToolsOnMachine: true + displayName: Build + + - task: PublishBuildArtifacts@1 + displayName: Publish Build BinLog + condition: always() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + ArtifactName: 'Windows Release build binlogs' + ArtifactType: Container + parallel: true + + # Windows With Compressed Metadata + - job: WindowsCompressedMetadata + variables: + - name: XUNIT_LOGS + value: $(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration) + - name: __VSNeverShowWhatsNew + value: 1 + pool: + # The PR build definition sets this variable: + # WindowsMachineQueueName=Windows.vs2022.amd64.open + # and there is an alternate build definition that sets this to a queue that is always scouting the + # next preview of Visual Studio. + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 120 + strategy: + maxParallel: 5 + matrix: + desktop_release: + _configuration: Release + _testKind: testDesktop + coreclr_release: + _configuration: Release + _testKind: testCoreclr + fsharpqa_release: + _configuration: Release + _testKind: testFSharpQA + vs_release: + _configuration: Release + _testKind: testVs + transparent_compiler_release: + _configuration: Release + _testKind: testCoreclr + + ${{ if eq(variables['Build.Reason'], 'Flaky, disabled, was PullRequest') }}: + inttests_release: + _configuration: Release + _testKind: testIntegration + steps: + - checkout: self + clean: true + + - powershell: eng\SetupVSHive.ps1 + displayName: Setup VS Hive + condition: or(eq(variables['_testKind'], 'testVs'), eq(variables['_testKind'], 'testIntegration')) + + # yes, this is miserable, but - https://github.com/dotnet/arcade/issues/13239 + - script: eng\CIBuild.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) + env: + NativeToolsOnMachine: true + displayName: Build / Test + condition: and( ne(variables['_testKind'], 'testIntegration'), ne(variables['System.JobName'], 'transparent_compiler_release') ) + + - script: eng\CIBuild.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) + env: + TEST_TRANSPARENT_COMPILER: 1 + NativeToolsOnMachine: true + displayName: Build / Test Transparent Compiler + condition: and( eq(variables['System.JobName'], 'transparent_compiler_release'), ne(variables['_testKind'], 'testIntegration') ) + + - script: eng\CIBuild.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) + env: + NativeToolsOnMachine: true + displayName: Build / Integration Test + continueOnError: true + condition: eq(variables['_testKind'], 'testIntegration') + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'NUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)' + continueOnError: true + condition: ne(variables['_testKind'], 'testFSharpQA') + - task: PublishBuildArtifacts@1 + displayName: Publish Tests BinLog + condition: always() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/$(_configuration)\Build.VisualFSharp.sln.binlog' + ArtifactName: 'Windows $(_configuration) $(_testKind) test binlogs' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration)' + ArtifactName: 'Windows $(_configuration) $(_testKind) test logs' + publishLocation: Container + continueOnError: true + condition: always() + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents Windows $(_testKind)' + publishLocation: Container + continueOnError: true + condition: failed() + + # Mock official build + - job: MockOfficial + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + steps: + - checkout: self + clean: true + - pwsh: .\eng\MockBuild.ps1 + displayName: Build with OfficialBuildId + + # Linux + - job: Linux + pool: + vmImage: $(UbuntuMachineQueueName) + variables: + - name: _SignType + value: Test + steps: + - checkout: self + clean: true + - script: ./eng/cibuild.sh --configuration $(_BuildConfig) --testcoreclr + displayName: Build / Test + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'NUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + continueOnError: true + condition: always() + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + ArtifactName: 'Linux $(_BuildConfig) test logs' + publishLocation: Container + continueOnError: true + condition: failed() + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents Linux' + publishLocation: Container + continueOnError: true + condition: failed() + + # MacOS + - job: MacOS + pool: + vmImage: macos-11 + variables: + - name: _SignType + value: Test + steps: + - checkout: self + clean: true + - script: ./eng/cibuild.sh --configuration $(_BuildConfig) --testcoreclr + env: + COMPlus_DefaultStackSize: 1000000 + displayName: Build / Test + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'NUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + continueOnError: true + condition: always() + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + ArtifactName: 'MacOS $(_BuildConfig) test logs' + publishLocation: Container + continueOnError: true + condition: failed() + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/artifacts/NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents Mac' + publishLocation: Container + continueOnError: true + condition: failed() + + # End to end build + - job: EndToEndBuildTests + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + strategy: + maxParallel: 2 + matrix: + regular: + _experimental_flag: '' + experimental_features: + _experimental_flag: '' + steps: + - checkout: self + clean: true + - script: .\Build.cmd -c Release -pack + env: + NativeToolsOnMachine: true + FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) + - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c Release + env: + FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) + displayName: End to end build tests + + # Up-to-date - disabled due to it being flaky + #- job: UpToDate_Windows + # pool: + # vmImage: windows-latest + # steps: + # - checkout: self + # clean: true + # - task: PowerShell@2 + # displayName: Run up-to-date build check + # inputs: + # filePath: eng\tests\UpToDate.ps1 + # arguments: -configuration $(_BuildConfig) -ci -binaryLog + + # Run Build with Fsharp Experimental Features + # Possible change: --times:$(Build.SourcesDirectory)/artifacts/log/Release/compiler_timing.csv + + # Plain FCS build Windows + - job: Plain_Build_Windows + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + variables: + - name: _BuildConfig + value: Debug + steps: + - checkout: self + clean: true + - script: dotnet --list-sdks + displayName: Report dotnet SDK versions + - task: UseDotNet@2 + displayName: install SDK + inputs: + packageType: sdk + useGlobalJson: true + includePreviewVersions: true + workingDirectory: $(Build.SourcesDirectory) + installationPath: $(Agent.ToolsDirectory)/dotnet + - script: dotnet build .\FSharp.Compiler.Service.sln /bl:\"artifacts/log/$(_BuildConfig)/ServiceRegularBuild.binlog\" + workingDirectory: $(Build.SourcesDirectory) + displayName: Regular rebuild of FSharp.Compiler.Service.sln + continueOnError: true + condition: always() + + # Plain FCS build Linux + - job: Plain_Build_Linux + pool: + vmImage: $(UbuntuMachineQueueName) + variables: + - name: _BuildConfig + value: Debug + steps: + - checkout: self + clean: true + - script: dotnet --list-sdks + displayName: Report dotnet SDK versions + - task: UseDotNet@2 + displayName: install SDK + inputs: + packageType: sdk + useGlobalJson: true + includePreviewVersions: true + workingDirectory: $(Build.SourcesDirectory) + installationPath: $(Agent.ToolsDirectory)/dotnet + - script: dotnet build ./FSharp.Compiler.Service.sln /bl:\"artifacts/log/$(_BuildConfig)/ServiceRegularBuild.binlog\" + workingDirectory: $(Build.SourcesDirectory) + displayName: Regular rebuild of FSharp.Compiler.Service.sln + continueOnError: true + condition: always() + + # Plain FCS build Mac + - job: Plain_Build_MacOS + pool: + vmImage: macos-11 + variables: + - name: _BuildConfig + value: Debug + steps: + - checkout: self + clean: true + - script: dotnet --list-sdks + displayName: Report dotnet SDK versions + - task: UseDotNet@2 + displayName: install SDK + inputs: + packageType: sdk + useGlobalJson: true + includePreviewVersions: true + workingDirectory: $(Build.SourcesDirectory) + installationPath: $(Agent.ToolsDirectory)/dotnet + - script: dotnet build ./FSharp.Compiler.Service.sln /bl:\"artifacts/log/$(_BuildConfig)/ServiceRegularBuild.binlog\" + workingDirectory: $(Build.SourcesDirectory) + displayName: Regular rebuild of FSharp.Compiler.Service.sln + continueOnError: true + condition: always() + + # Build and run fast benchmarks + - job: Benchmarks + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + variables: + - name: _BuildConfig + value: Release + steps: + - checkout: self + clean: true + - script: eng\CIBuild.cmd -configuration $(_BuildConfig) -testBenchmarks + displayName: Smoke test fast benchmarks + continueOnError: true + condition: always() + + # Test trimming on Windows + - job: Build_And_Test_AOT_Windows + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + strategy: + maxParallel: 2 + matrix: + compressed_metadata: + _kind: "-compressAllMetadata" + classic_metadata: + _kind: "" + variables: + - name: _BuildConfig + value: Release + steps: + - checkout: self + clean: true + - task: UseDotNet@2 + displayName: install SDK + inputs: + packageType: sdk + useGlobalJson: true + includePreviewVersions: true + workingDirectory: $(Build.SourcesDirectory) + installationPath: $(Agent.ToolsDirectory)/dotnet + - script: dotnet --list-sdks + displayName: Report dotnet SDK versions + - script: .\Build.cmd $(_kind) -pack -c $(_BuildConfig) + env: + NativeToolsOnMachine: true + displayName: Initial build and prepare packages. + - script: $(Build.SourcesDirectory)/tests/AheadOfTime/check.cmd + displayName: Build, trim, publish and check the state of the trimmed app. + workingDirectory: $(Build.SourcesDirectory)/tests/AheadOfTime + - task: PublishPipelineArtifact@1 + displayName: Publish Trim Tests Logs + inputs: + targetPath: './artifacts/log/Release/AheadOfTime/Trimming/' + artifactName: 'Trim Test Logs Attempt $(System.JobAttempt) Logs $(_kind)' + continueOnError: true + condition: always() + + # Arcade-powered source build + # turned off until https://github.com/dotnet/source-build/issues/1795 is fixed + # - template: /eng/common/templates/jobs/jobs.yml + # parameters: + # enablePublishUsingPipelines: true + # enablePublishBuildArtifacts: true + # enablePublishBuildAssets: true + # artifacts: + # publish: + # artifacts: true + # manifests: true + # runSourceBuild: true + # sourceBuildParameters: + # includeDefaultManagedPlatform: true + +#---------------------------------------------------------------------------------------------------------------------# +# Post Build # +#---------------------------------------------------------------------------------------------------------------------# +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: eng/common/templates/post-build/post-build.yml + parameters: + publishingInfraVersion: 3 + # Symbol validation is not entirely reliable as of yet, so should be turned off until https://github.com/dotnet/arcade/issues/2871 is resolved. + enableSymbolValidation: false + # SourceLink improperly looks for generated files. See https://github.com/dotnet/arcade/issues/3069 + enableSourceLinkValidation: false + # Enable SDL validation, passing through values from the 'DotNet-FSharp-SDLValidation-Params' group. + SDLValidationParameters: + enable: true + params: >- + -SourceToolsList @("policheck","credscan") + -ArtifactToolsList @("binskim") + -BinskimAdditionalRunConfigParams @("IgnorePdbLoadError < True","Recurse < True") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName "FSharp" + -TsaCodebaseName "FSharp-GitHub" + -TsaPublish $True + -PoliCheckAdditionalRunConfigParams @("UserExclusionPath < $(Build.SourcesDirectory)/eng/policheck_exclusions.xml") + +#---------------------------------------------------------------------------------------------------------------------# +# VS Insertion # +#---------------------------------------------------------------------------------------------------------------------# +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: eng/release/insert-into-vs.yml + parameters: + componentBranchName: refs/heads/release/dev17.9 + insertTargetBranch: rel/d17.9 + insertTeamEmail: fsharpteam@microsoft.com + insertTeamName: 'F#' + completeInsertion: 'auto' From 2c703e5e5bf01d0a8a7acf59c3e4da3efc8cefab Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 17:04:48 +0100 Subject: [PATCH 13/34] Remove signed build from PR definition --- azure-pipelines-PR.yml | 203 ++--------------------------------------- azure-pipelines.yml | 8 +- 2 files changed, 13 insertions(+), 198 deletions(-) diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index a680b0b557..d1786cac4e 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -49,8 +49,12 @@ pr: variables: - name: _TeamName value: FSharp + - name: TeamName + value: FSharp - name: _BuildConfig value: Release + - name: _SignType + value: Real - name: _PublishUsingPipelines value: true - name: _DotNetArtifactsCategory @@ -59,135 +63,18 @@ variables: value: Products/$(System.TeamProject)/$(Build.Repository.Name)/$(Build.SourceBranchName)/$(Build.BuildNumber) - name: Codeql.Enabled value: true - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - name: _DotNetValidationArtifactsCategory - value: .NETCoreValidation - - group: DotNet-FSharp-SDLValidation-Params - - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: - - name: RunningAsPullRequest - value: true - # Pick up pool provider name behavior from shared yaml template + - name: _DotNetValidationArtifactsCategory + value: .NETCoreValidation + - group: DotNet-FSharp-SDLValidation-Params - template: /eng/common/templates/variables/pool-providers.yml - -# Variables defined in yml cannot be overridden at queue time; instead overridable variables must be defined in the web UI. -# Commenting out until something like this is supported: https://github.com/Microsoft/azure-pipelines-yaml/pull/129 -#variables: -#- name: SkipTests -# defaultValue: false + - name: RunningAsPullRequest + value: true stages: - stage: build displayName: Build jobs: - #-------------------------------------------------------------------------------------------------------------------# - # Signed build # - #-------------------------------------------------------------------------------------------------------------------# - - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/release/dev17.9') }}: - - template: /eng/common/templates/job/onelocbuild.yml - parameters: - MirrorRepo: fsharp - MirrorBranch: release/dev17.9 - LclSource: lclFilesfromPackage - LclPackageId: 'LCL-JUNO-PROD-FSHARP' - - template: /eng/common/templates/jobs/jobs.yml - parameters: - enableMicrobuild: true - enablePublishBuildArtifacts: true - enablePublishTestResults: false - enablePublishBuildAssets: true - enablePublishUsingPipelines: $(_PublishUsingPipelines) - enableSourceBuild: true - enableTelemetry: true - helixRepo: dotnet/fsharp - jobs: - - job: Full_Signed - pool: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022preview.amd64 - timeoutInMinutes: 300 - variables: - - group: DotNet-Symbol-Server-Pats - - group: DotNet-DevDiv-Insertion-Workflow-Variables - - name: _SignType - value: Real - steps: - - checkout: self - clean: true - - template: /eng/restore-internal-tools.yml - - script: eng\CIBuild.cmd - -configuration $(_BuildConfig) - -prepareMachine - -testAllButIntegration - -officialSkipTests $(SkipTests) - /p:SignType=$(_SignType) - /p:DotNetSignType=$(_SignType) - /p:MicroBuild_SigningEnabled=true - /p:OverridePackageSource=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json - /p:TeamName=$(_TeamName) - /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) - /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) - /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) - /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) - /p:OfficialBuildId=$(BUILD.BUILDNUMBER) - /p:PublishToSymbolServer=true - /p:VisualStudioDropName=$(VisualStudioDropName) - /p:GenerateSbom=true - env: - NativeToolsOnMachine: true - - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c $(_BuildConfig) - displayName: End to end build tests - continueOnError: true - condition: always() - - task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFormat: 'NUnit' - testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' - continueOnError: true - condition: ne(variables['SkipTests'], 'true') - - task: PublishBuildArtifacts@1 - displayName: Publish Test Logs - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig)' - ArtifactName: 'Test Logs' - publishLocation: Container - continueOnError: true - condition: ne(variables['SkipTests'], 'true') - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact Packages - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig)' - ArtifactName: 'Packages' - condition: succeeded() - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact VSSetup - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' - ArtifactName: 'VSSetup' - condition: succeeded() - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact Nightly - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpDebug.vsix' - ArtifactName: 'Nightly' - condition: succeeded() - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact Symbols - inputs: - PathtoPublish: '$(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig)' - ArtifactName: 'NativeSymbols' - condition: succeeded() - - task: ms-vseng.MicroBuildTasks.4305a8de-ba66-4d8b-b2d1-0dc4ecbbf5e8.MicroBuildUploadVstsDropFolder@1 - displayName: Upload VSTS Drop - inputs: - DropName: $(VisualStudioDropName) - DropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' - AccessToken: $(dn-bot-devdiv-drop-rw-code-rw) - condition: succeeded() - #-------------------------------------------------------------------------------------------------------------------# # PR builds without logs publishing # #-------------------------------------------------------------------------------------------------------------------# @@ -680,22 +567,6 @@ stages: FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) displayName: End to end build tests - # Up-to-date - disabled due to it being flaky - #- job: UpToDate_Windows - # pool: - # vmImage: windows-latest - # steps: - # - checkout: self - # clean: true - # - task: PowerShell@2 - # displayName: Run up-to-date build check - # inputs: - # filePath: eng\tests\UpToDate.ps1 - # arguments: -configuration $(_BuildConfig) -ci -binaryLog - - # Run Build with Fsharp Experimental Features - # Possible change: --times:$(Build.SourcesDirectory)/artifacts/log/Release/compiler_timing.csv - # Plain FCS build Windows - job: Plain_Build_Windows pool: @@ -833,59 +704,3 @@ stages: artifactName: 'Trim Test Logs Attempt $(System.JobAttempt) Logs $(_kind)' continueOnError: true condition: always() - - # Arcade-powered source build - # turned off until https://github.com/dotnet/source-build/issues/1795 is fixed - # - template: /eng/common/templates/jobs/jobs.yml - # parameters: - # enablePublishUsingPipelines: true - # enablePublishBuildArtifacts: true - # enablePublishBuildAssets: true - # artifacts: - # publish: - # artifacts: true - # manifests: true - # runSourceBuild: true - # sourceBuildParameters: - # includeDefaultManagedPlatform: true - -#---------------------------------------------------------------------------------------------------------------------# -# Post Build # -#---------------------------------------------------------------------------------------------------------------------# -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: eng/common/templates/post-build/post-build.yml - parameters: - publishingInfraVersion: 3 - # Symbol validation is not entirely reliable as of yet, so should be turned off until https://github.com/dotnet/arcade/issues/2871 is resolved. - enableSymbolValidation: false - # SourceLink improperly looks for generated files. See https://github.com/dotnet/arcade/issues/3069 - enableSourceLinkValidation: false - # Enable SDL validation, passing through values from the 'DotNet-FSharp-SDLValidation-Params' group. - SDLValidationParameters: - enable: true - params: >- - -SourceToolsList @("policheck","credscan") - -ArtifactToolsList @("binskim") - -BinskimAdditionalRunConfigParams @("IgnorePdbLoadError < True","Recurse < True") - -TsaInstanceURL $(_TsaInstanceURL) - -TsaProjectName $(_TsaProjectName) - -TsaNotificationEmail $(_TsaNotificationEmail) - -TsaCodebaseAdmin $(_TsaCodebaseAdmin) - -TsaBugAreaPath $(_TsaBugAreaPath) - -TsaIterationPath $(_TsaIterationPath) - -TsaRepositoryName "FSharp" - -TsaCodebaseName "FSharp-GitHub" - -TsaPublish $True - -PoliCheckAdditionalRunConfigParams @("UserExclusionPath < $(Build.SourcesDirectory)/eng/policheck_exclusions.xml") - -#---------------------------------------------------------------------------------------------------------------------# -# VS Insertion # -#---------------------------------------------------------------------------------------------------------------------# -- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: eng/release/insert-into-vs.yml - parameters: - componentBranchName: refs/heads/release/dev17.9 - insertTargetBranch: rel/d17.9 - insertTeamEmail: fsharpteam@microsoft.com - insertTeamName: 'F#' - completeInsertion: 'auto' diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5e01f066d6..29425edeb6 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,9 +26,9 @@ trigger: variables: - name: FSharpReleaseBranchName - value: 1espm + value: 1espm # Should be 'current' branch name, i.e. 'main' in main or 'release/dev17.10' in release/dev17.10 - name: VSInsertionTargetBranchName - value: main + value: main # Should be 'previous' release branch in main and current in release, i.e. 'rel/d17.10' in 'release/dev17.10' - name: _TeamName value: FSharp - name: TeamName @@ -71,7 +71,7 @@ extends: enabled: false # VS SBOM is generated with other steps justificationForDisabling: 'SBOM for F# is generated via build process.' sourceAnalysisPool: - name: $(DncEngInternalBuildPool) + name: NetCore1ESPool-Svc-Internal image: 1es-windows-2022-pt os: windows stages: @@ -81,7 +81,7 @@ extends: #-------------------------------------------------------------------------------------------------------------------# # Signed build # #-------------------------------------------------------------------------------------------------------------------# - # Localization: we only run it for specific release branches. + # Localization: we only run it for specific release branches - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/release/dev17.9') }}: - template: /eng/common/templates-official/job/onelocbuild.yml@self parameters: From 6ec950d14b162cf9afe14616bb77239ae5cb9a67 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 18:43:56 +0100 Subject: [PATCH 14/34] Remove signed build from PR definition --- azure-pipelines-PR.yml | 205 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 196 insertions(+), 9 deletions(-) diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index d1786cac4e..713a654c34 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -3,6 +3,7 @@ trigger: branches: include: - main + - dev16.1 - feature/* - release/* paths: @@ -28,6 +29,7 @@ pr: branches: include: - main + - dev16.1 - feature/* - release/* paths: @@ -49,12 +51,8 @@ pr: variables: - name: _TeamName value: FSharp - - name: TeamName - value: FSharp - name: _BuildConfig value: Release - - name: _SignType - value: Real - name: _PublishUsingPipelines value: true - name: _DotNetArtifactsCategory @@ -63,18 +61,135 @@ variables: value: Products/$(System.TeamProject)/$(Build.Repository.Name)/$(Build.SourceBranchName)/$(Build.BuildNumber) - name: Codeql.Enabled value: true - - name: _DotNetValidationArtifactsCategory - value: .NETCoreValidation - - group: DotNet-FSharp-SDLValidation-Params + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - name: _DotNetValidationArtifactsCategory + value: .NETCoreValidation + - group: DotNet-FSharp-SDLValidation-Params + - ${{ if and(eq(variables['System.TeamProject'], 'public'), eq(variables['Build.Reason'], 'PullRequest')) }}: + - name: RunningAsPullRequest + value: true + # Pick up pool provider name behavior from shared yaml template - template: /eng/common/templates/variables/pool-providers.yml - - name: RunningAsPullRequest - value: true + +# Variables defined in yml cannot be overridden at queue time; instead overridable variables must be defined in the web UI. +# Commenting out until something like this is supported: https://github.com/Microsoft/azure-pipelines-yaml/pull/129 +#variables: +#- name: SkipTests +# defaultValue: false stages: - stage: build displayName: Build jobs: + #-------------------------------------------------------------------------------------------------------------------# + # Signed build # + #-------------------------------------------------------------------------------------------------------------------# + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/release/dev17.9') }}: + - template: /eng/common/templates/job/onelocbuild.yml + parameters: + MirrorRepo: fsharp + MirrorBranch: release/dev17.9 + LclSource: lclFilesfromPackage + LclPackageId: 'LCL-JUNO-PROD-FSHARP' + - template: /eng/common/templates/jobs/jobs.yml + parameters: + enableMicrobuild: true + enablePublishBuildArtifacts: true + enablePublishTestResults: false + enablePublishBuildAssets: true + enablePublishUsingPipelines: $(_PublishUsingPipelines) + enableSourceBuild: true + enableTelemetry: true + helixRepo: dotnet/fsharp + jobs: + - job: Full_Signed + pool: + name: $(DncEngInternalBuildPool) + demands: ImageOverride -equals windows.vs2022preview.amd64 + timeoutInMinutes: 300 + variables: + - group: DotNet-Symbol-Server-Pats + - group: DotNet-DevDiv-Insertion-Workflow-Variables + - name: _SignType + value: Real + steps: + - checkout: self + clean: true + - template: /eng/restore-internal-tools.yml + - script: eng\CIBuild.cmd + -configuration $(_BuildConfig) + -prepareMachine + -testAllButIntegration + -officialSkipTests $(SkipTests) + /p:SignType=$(_SignType) + /p:DotNetSignType=$(_SignType) + /p:MicroBuild_SigningEnabled=true + /p:OverridePackageSource=https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + /p:TeamName=$(_TeamName) + /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines) + /p:DotNetArtifactsCategory=$(_DotNetArtifactsCategory) + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:OfficialBuildId=$(BUILD.BUILDNUMBER) + /p:PublishToSymbolServer=true + /p:VisualStudioDropName=$(VisualStudioDropName) + /p:GenerateSbom=true + env: + NativeToolsOnMachine: true + - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c $(_BuildConfig) + displayName: End to end build tests + continueOnError: true + condition: always() + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'NUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + continueOnError: true + condition: ne(variables['SkipTests'], 'true') + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig)' + ArtifactName: 'Test Logs' + publishLocation: Container + continueOnError: true + condition: ne(variables['SkipTests'], 'true') + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact Packages + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig)' + ArtifactName: 'Packages' + condition: succeeded() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact VSSetup + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' + ArtifactName: 'VSSetup' + condition: succeeded() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact Nightly + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpDebug.vsix' + ArtifactName: 'Nightly' + condition: succeeded() + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact Symbols + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig)' + ArtifactName: 'NativeSymbols' + condition: succeeded() + - task: ms-vseng.MicroBuildTasks.4305a8de-ba66-4d8b-b2d1-0dc4ecbbf5e8.MicroBuildUploadVstsDropFolder@1 + displayName: Upload VSTS Drop + inputs: + DropName: $(VisualStudioDropName) + DropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' + AccessToken: $(dn-bot-devdiv-drop-rw-code-rw) + condition: succeeded() + #-------------------------------------------------------------------------------------------------------------------# # PR builds without logs publishing # #-------------------------------------------------------------------------------------------------------------------# @@ -567,6 +682,22 @@ stages: FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) displayName: End to end build tests + # Up-to-date - disabled due to it being flaky + #- job: UpToDate_Windows + # pool: + # vmImage: windows-latest + # steps: + # - checkout: self + # clean: true + # - task: PowerShell@2 + # displayName: Run up-to-date build check + # inputs: + # filePath: eng\tests\UpToDate.ps1 + # arguments: -configuration $(_BuildConfig) -ci -binaryLog + + # Run Build with Fsharp Experimental Features + # Possible change: --times:$(Build.SourcesDirectory)/artifacts/log/Release/compiler_timing.csv + # Plain FCS build Windows - job: Plain_Build_Windows pool: @@ -704,3 +835,59 @@ stages: artifactName: 'Trim Test Logs Attempt $(System.JobAttempt) Logs $(_kind)' continueOnError: true condition: always() + + # Arcade-powered source build + # turned off until https://github.com/dotnet/source-build/issues/1795 is fixed + # - template: /eng/common/templates/jobs/jobs.yml + # parameters: + # enablePublishUsingPipelines: true + # enablePublishBuildArtifacts: true + # enablePublishBuildAssets: true + # artifacts: + # publish: + # artifacts: true + # manifests: true + # runSourceBuild: true + # sourceBuildParameters: + # includeDefaultManagedPlatform: true + +#---------------------------------------------------------------------------------------------------------------------# +# Post Build # +#---------------------------------------------------------------------------------------------------------------------# +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: eng/common/templates/post-build/post-build.yml + parameters: + publishingInfraVersion: 3 + # Symbol validation is not entirely reliable as of yet, so should be turned off until https://github.com/dotnet/arcade/issues/2871 is resolved. + enableSymbolValidation: false + # SourceLink improperly looks for generated files. See https://github.com/dotnet/arcade/issues/3069 + enableSourceLinkValidation: false + # Enable SDL validation, passing through values from the 'DotNet-FSharp-SDLValidation-Params' group. + SDLValidationParameters: + enable: true + params: >- + -SourceToolsList @("policheck","credscan") + -ArtifactToolsList @("binskim") + -BinskimAdditionalRunConfigParams @("IgnorePdbLoadError < True","Recurse < True") + -TsaInstanceURL $(_TsaInstanceURL) + -TsaProjectName $(_TsaProjectName) + -TsaNotificationEmail $(_TsaNotificationEmail) + -TsaCodebaseAdmin $(_TsaCodebaseAdmin) + -TsaBugAreaPath $(_TsaBugAreaPath) + -TsaIterationPath $(_TsaIterationPath) + -TsaRepositoryName "FSharp" + -TsaCodebaseName "FSharp-GitHub" + -TsaPublish $True + -PoliCheckAdditionalRunConfigParams @("UserExclusionPath < $(Build.SourcesDirectory)/eng/policheck_exclusions.xml") + +#---------------------------------------------------------------------------------------------------------------------# +# VS Insertion # +#---------------------------------------------------------------------------------------------------------------------# +- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - template: eng/release/insert-into-vs.yml + parameters: + componentBranchName: refs/heads/release/dev17.9 + insertTargetBranch: rel/d17.9 + insertTeamEmail: fsharpteam@microsoft.com + insertTeamName: 'F#' + completeInsertion: 'auto' \ No newline at end of file From 11381cf610fdde56e398aa177f1f695a125143b1 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 19:21:42 +0100 Subject: [PATCH 15/34] Try fix insertion --- azure-pipelines.yml | 13 +++++++------ eng/release/insert-into-vs.yml | 14 +++++++++----- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 29425edeb6..b107d747b2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -117,7 +117,7 @@ extends: displayName: 'Publish Test Logs' condition: and(succeededOrFailed(), ne(variables['SkipTests'], 'true')) targetPath: $(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig) - artifactName: 'Test Logs' + artifactName: TestLogs - output: pipelineArtifact displayName: 'Publish Artifact Packages' condition: succeeded() @@ -127,17 +127,17 @@ extends: displayName: 'Publish Artifact VSSetup' condition: succeeded() targetPath: $(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion - artifactName: 'VSSetup' + artifactName: VSSetup - output: pipelineArtifact displayName: 'Publish Nightly VSIX' condition: succeeded() targetPath: $(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpDebug.vsix - artifactName: 'Nightly' + artifactName: Nightly - output: pipelineArtifact displayName: 'Publish Artifact Symbols' condition: succeeded() targetPath: $(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig) - artifactName: 'NativeSymbols' + artifactName: NativeSymbols - output: microBuildVstsDrop dropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' dropName: $(VisualStudioDropName) @@ -179,7 +179,7 @@ extends: #---------------------------------------------------------------------------------------------------------------------# # Post Build # #---------------------------------------------------------------------------------------------------------------------# - - template: eng/common/templates-official/post-build/post-build.yml + - template: eng/common/templates-official/post-build/post-build.yml@self parameters: publishingInfraVersion: 3 # Symbol validation is not entirely reliable as of yet, so should be turned off until https://github.com/dotnet/arcade/issues/2871 is resolved. @@ -207,7 +207,8 @@ extends: #---------------------------------------------------------------------------------------------------------------------# # VS Insertion # #---------------------------------------------------------------------------------------------------------------------# - - template: eng/release/insert-into-vs.yml + + - template: eng/release/insert-into-vs.yml@self parameters: componentBranchName: refs/heads/$(FSharpReleaseBranchName) insertTargetBranch: $(VSInsertionBranchName) diff --git a/eng/release/insert-into-vs.yml b/eng/release/insert-into-vs.yml index 746b73d7c6..371d2adf01 100644 --- a/eng/release/insert-into-vs.yml +++ b/eng/release/insert-into-vs.yml @@ -12,6 +12,15 @@ stages: dependsOn: publish_using_darc displayName: Insert into VS jobs: + - job: Download_VSSetup + displayName: Download VSSetup + templateContext: + # The type of the job should be specified as artifactJob + type: artifactJob + inputs: + - input: pipelineArtifact + artifactName: VSSetup + targetPath: $(Build.ArtifactStagingDirectory)/VSSetup - job: Insert_VS pool: name: NetCore1ESPool-Svc-Internal @@ -31,11 +40,6 @@ stages: - name: InsertPayloadName value: '${{ parameters.insertTeamName }} $(Build.SourceBranchName) $(Build.BuildNumber)' steps: - - task: DownloadBuildArtifacts@0 - displayName: Download Insertion Artifacts - inputs: - buildType: current - artifactName: VSSetup - task: PowerShell@2 displayName: Get Publish URLs inputs: From 249495e4f31cf6cf354a72d3bb0f865de2595344 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 19:27:41 +0100 Subject: [PATCH 16/34] Try fix insertion --- eng/release/insert-into-vs.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eng/release/insert-into-vs.yml b/eng/release/insert-into-vs.yml index 371d2adf01..3e9ac07d7b 100644 --- a/eng/release/insert-into-vs.yml +++ b/eng/release/insert-into-vs.yml @@ -21,6 +21,10 @@ stages: - input: pipelineArtifact artifactName: VSSetup targetPath: $(Build.ArtifactStagingDirectory)/VSSetup + outputs: + - output: pipelineArtifact + artifactName: VSSetup + targetPath: $(Build.ArtifactStagingDirectory)/VSSetup - job: Insert_VS pool: name: NetCore1ESPool-Svc-Internal From 4e42a4469cc321b86d1c8a7d728af4a4d093d8ed Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 19:29:55 +0100 Subject: [PATCH 17/34] Try artifact job to share artifact between stages --- eng/release/insert-into-vs.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/release/insert-into-vs.yml b/eng/release/insert-into-vs.yml index 3e9ac07d7b..8204fa1b32 100644 --- a/eng/release/insert-into-vs.yml +++ b/eng/release/insert-into-vs.yml @@ -15,7 +15,6 @@ stages: - job: Download_VSSetup displayName: Download VSSetup templateContext: - # The type of the job should be specified as artifactJob type: artifactJob inputs: - input: pipelineArtifact From 102cfd79ad03df9b091e460577db0dcdd829339b Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Wed, 6 Mar 2024 19:29:55 +0100 Subject: [PATCH 18/34] Try artifact job to share artifact between stages --- azure-pipelines.yml | 1 - eng/release/insert-into-vs.yml | 13 ------------- 2 files changed, 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b107d747b2..1982138ca5 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -207,7 +207,6 @@ extends: #---------------------------------------------------------------------------------------------------------------------# # VS Insertion # #---------------------------------------------------------------------------------------------------------------------# - - template: eng/release/insert-into-vs.yml@self parameters: componentBranchName: refs/heads/$(FSharpReleaseBranchName) diff --git a/eng/release/insert-into-vs.yml b/eng/release/insert-into-vs.yml index 3e9ac07d7b..ecac3544c1 100644 --- a/eng/release/insert-into-vs.yml +++ b/eng/release/insert-into-vs.yml @@ -12,19 +12,6 @@ stages: dependsOn: publish_using_darc displayName: Insert into VS jobs: - - job: Download_VSSetup - displayName: Download VSSetup - templateContext: - # The type of the job should be specified as artifactJob - type: artifactJob - inputs: - - input: pipelineArtifact - artifactName: VSSetup - targetPath: $(Build.ArtifactStagingDirectory)/VSSetup - outputs: - - output: pipelineArtifact - artifactName: VSSetup - targetPath: $(Build.ArtifactStagingDirectory)/VSSetup - job: Insert_VS pool: name: NetCore1ESPool-Svc-Internal From aaacbf6aac6a7aa336a5833f2e8b26b55918d279 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Thu, 7 Mar 2024 12:55:17 +0100 Subject: [PATCH 19/34] Try downloading artifact with 'download' --- eng/release/insert-into-vs.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eng/release/insert-into-vs.yml b/eng/release/insert-into-vs.yml index ecac3544c1..854134c18f 100644 --- a/eng/release/insert-into-vs.yml +++ b/eng/release/insert-into-vs.yml @@ -31,6 +31,8 @@ stages: - name: InsertPayloadName value: '${{ parameters.insertTeamName }} $(Build.SourceBranchName) $(Build.BuildNumber)' steps: + - download: current + artifact: VSSetup - task: PowerShell@2 displayName: Get Publish URLs inputs: From 67694503a17fd81a9818c31393111cc703e946e4 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Thu, 7 Mar 2024 15:02:33 +0100 Subject: [PATCH 20/34] Upload artifact via taks --- azure-pipelines.yml | 72 +++++++++++++++++++++++++++++---------------- 1 file changed, 47 insertions(+), 25 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1982138ca5..d44805e636 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -113,31 +113,6 @@ extends: signType: $(_SignType) zipSources: false outputs: - - output: pipelineArtifact - displayName: 'Publish Test Logs' - condition: and(succeededOrFailed(), ne(variables['SkipTests'], 'true')) - targetPath: $(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig) - artifactName: TestLogs - - output: pipelineArtifact - displayName: 'Publish Artifact Packages' - condition: succeeded() - targetPath: $(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig) - artifactName: 'Packages' - - output: pipelineArtifact - displayName: 'Publish Artifact VSSetup' - condition: succeeded() - targetPath: $(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion - artifactName: VSSetup - - output: pipelineArtifact - displayName: 'Publish Nightly VSIX' - condition: succeeded() - targetPath: $(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpDebug.vsix - artifactName: Nightly - - output: pipelineArtifact - displayName: 'Publish Artifact Symbols' - condition: succeeded() - targetPath: $(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig) - artifactName: NativeSymbols - output: microBuildVstsDrop dropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' dropName: $(VisualStudioDropName) @@ -175,6 +150,53 @@ extends: /p:GenerateSbom=true env: NativeToolsOnMachine: true + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'NUnit' + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + continueOnError: true + condition: ne(variables['SkipTests'], 'true') + - task: 1ES.PublishPipelineArtifact@1 + displayName: Publish Test Logs + inputs: + targetPath: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_BuildConfig)' + artifactName: 'Test Logs' + publishLocation: Container + continueOnError: true + condition: ne(variables['SkipTests'], 'true') + - task: 1ES.PublishPipelineArtifact@1 + displayName: Publish Artifact Packages + inputs: + targetPath: '$(Build.SourcesDirectory)\artifacts\packages\$(_BuildConfig)' + artifactName: 'Packages' + condition: succeeded() + - task: 1ES.PublishPipelineArtifact@1 + displayName: Publish Artifact VSSetup + inputs: + targetPath: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' + artifactName: 'VSSetup' + condition: succeeded() + - task: 1ES.PublishPipelineArtifact@1 + displayName: Publish Artifact Nightly + inputs: + targetPath: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\VisualFSharpDebug.vsix' + artifactName: 'Nightly' + condition: succeeded() + - task: 1ES.PublishPipelineArtifact@1 + displayName: Publish Artifact Symbols + inputs: + targetPath: '$(Build.SourcesDirectory)\artifacts\SymStore\$(_BuildConfig)' + artifactName: 'NativeSymbols' + condition: succeeded() + - task: 1ES.MicroBuildVstsDrop@1 + displayName: Upload VSTS Drop + inputs: + dropName: $(VisualStudioDropName) + dropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' + accessToken: $(dn-bot-devdiv-drop-rw-code-rw) + condition: succeeded() #---------------------------------------------------------------------------------------------------------------------# # Post Build # From 17c5dfe404ef6474adaa14a0974e362ec3d31e55 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Mon, 11 Mar 2024 14:25:05 +0100 Subject: [PATCH 21/34] Adjust paths of VSSetup artefact + print out tree of the pipeline workspace --- eng/release/insert-into-vs.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/eng/release/insert-into-vs.yml b/eng/release/insert-into-vs.yml index 854134c18f..e15b691f59 100644 --- a/eng/release/insert-into-vs.yml +++ b/eng/release/insert-into-vs.yml @@ -33,21 +33,28 @@ stages: steps: - download: current artifact: VSSetup + - task: PowerShell@2 + displayName: List Workspace + inputs: + targetType: 'inline' + pwsh: true + script: | + Tree $(Pipeline.Workspace) /F - task: PowerShell@2 displayName: Get Publish URLs inputs: filePath: $(Build.SourcesDirectory)/eng/release/scripts/GetPublishUrls.ps1 - arguments: -accessToken $(System.AccessToken) -buildId $(Build.BuildId) -insertionDir $(Build.ArtifactStagingDirectory)\VSSetup + arguments: -accessToken $(System.AccessToken) -buildId $(Build.BuildId) -insertionDir $(Pipeline.Workspace)\VSSetup - task: PowerShell@2 displayName: Get versions for default.config inputs: filePath: $(Build.SourcesDirectory)/eng/release/scripts/GetDefaultConfigVersions.ps1 - arguments: -packagesDir $(Build.ArtifactStagingDirectory)\VSSetup\DevDivPackages + arguments: -packagesDir $(Pipeline.Workspace)\VSSetup\DevDivPackages - task: PowerShell@2 displayName: Get versions for AssemblyVersions.tt inputs: filePath: $(Build.SourcesDirectory)/eng/release/scripts/GetAssemblyVersions.ps1 - arguments: -assemblyVersionsPath $(Build.ArtifactStagingDirectory)\VSSetup\DevDivPackages\DependentAssemblyVersions.csv + arguments: -assemblyVersionsPath $(Pipeline.Workspace)\VSSetup\DevDivPackages\DependentAssemblyVersions.csv - task: PowerShell@2 displayName: Calculate autocompletion state inputs: From 90f4e0950b6f8d41b3092f5e7fbd2de8a9f317c8 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Mon, 11 Mar 2024 15:25:35 +0100 Subject: [PATCH 22/34] Adjust VSTS drops upload --- azure-pipelines.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d44805e636..a93b8021ea 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -112,14 +112,6 @@ extends: feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json signType: $(_SignType) zipSources: false - outputs: - - output: microBuildVstsDrop - dropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' - dropName: $(VisualStudioDropName) - dropRetentionDays: 90 - accessToken: $(dn-bot-devdiv-drop-rw-code-rw) - dropServiceUri: 'https://devdiv.artifacts.visualstudio.com' - vsDropServiceUri: 'https://vsdrop.corp.microsoft.com/file/v1' variables: - group: DotNet-Symbol-Server-Pats @@ -195,7 +187,11 @@ extends: inputs: dropName: $(VisualStudioDropName) dropFolder: '$(Build.SourcesDirectory)\artifacts\VSSetup\$(_BuildConfig)\Insertion' + dropRetentionDays: 90 accessToken: $(dn-bot-devdiv-drop-rw-code-rw) + dropServiceUri: 'https://devdiv.artifacts.visualstudio.com' + vsDropServiceUri: 'https://vsdrop.corp.microsoft.com/file/v1' + condition: succeeded() #---------------------------------------------------------------------------------------------------------------------# From 9469af53f17541dca407f10492b2a20792ef8304 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Mon, 11 Mar 2024 17:56:46 +0100 Subject: [PATCH 23/34] Restore Arcade's version of SetupNugetSources.ps1 --- eng/common/SetupNugetSources.ps1 | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 6a05966d57..efa2fd72bf 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -25,7 +25,7 @@ [CmdletBinding()] param ( [Parameter(Mandatory = $true)][string]$ConfigFile, - [Parameter(Mandatory = $true)][SecureString]$Password + [Parameter(Mandatory = $true)][string]$Password ) $ErrorActionPreference = "Stop" @@ -35,7 +35,7 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 # Add source entry to PackageSources -function AddPackageSource($sources, $SourceName, $SourceEndPoint, [SecureString] $creds, $Username, [SecureString] $Password) { +function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) @@ -48,12 +48,11 @@ function AddPackageSource($sources, $SourceName, $SourceEndPoint, [SecureString] else { Write-Host "Package source $SourceName already present." } - - AddCredential -Creds $creds -Source $SourceName -Username $Username -Password $Password + AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd } # Add a credential node for the specified source -function AddCredential([SecureString] $creds, $source, $username, [SecureString] $password) { +function AddCredential($creds, $source, $username, $pwd) { # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -82,17 +81,18 @@ function AddCredential([SecureString] $creds, $source, $username, [SecureString] $passwordElement.SetAttribute("key", "ClearTextPassword") $sourceElement.AppendChild($passwordElement) | Out-Null } - $passwordElement.SetAttribute("value", $Password) + + $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, [SecureString] $Creds, $Username, [SecureString] $Password) { +function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." ForEach ($PackageSource in $maestroPrivateSources) { Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -Password $Password + AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd } } @@ -144,13 +144,13 @@ if ($disabledSources -ne $null) { $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password +InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password # 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } $dotnetVersions = @('5','6','7','8') @@ -159,9 +159,9 @@ foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password } } -$doc.Save($filename) +$doc.Save($filename) \ No newline at end of file From d4c97a4c00f4ebebf531be6c04598beffb9466fc Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Mon, 11 Mar 2024 18:10:24 +0100 Subject: [PATCH 24/34] Added more explaination for F# branches --- azure-pipelines.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a93b8021ea..80d95f96db 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -25,10 +25,16 @@ trigger: - TESTGUIDE.md variables: + # Release branch for F# + # Should be 'current' branch name, i.e. 'main' in dotnet/fsharp/refs/main or 'release/dev17.10' in dotnet/fsharp/refs/release/dev17.10 - name: FSharpReleaseBranchName - value: 1espm # Should be 'current' branch name, i.e. 'main' in main or 'release/dev17.10' in release/dev17.10 + value: 1espm + # VS Insertion branch name + # Should be previous release branch in 'main' and 'main' in release branch + # (since for all *new* release branches we insert into VS main and for all *previous* releases we insert into corresponding VS release), + # i.e. 'rel/d17.9' in dotnet/fsharp/refs/main and 'main' in F# dotnet/fsharp/refs/release/dev17.10 - name: VSInsertionTargetBranchName - value: main # Should be 'previous' release branch in main and current in release, i.e. 'rel/d17.10' in 'release/dev17.10' + value: main - name: _TeamName value: FSharp - name: TeamName From 0c7277d9bd824edb25c66d8ed8e8b842758b71e0 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Mon, 11 Mar 2024 18:10:47 +0100 Subject: [PATCH 25/34] Added more explaination for F# branches --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 80d95f96db..d8bba26a76 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,7 +29,7 @@ variables: # Should be 'current' branch name, i.e. 'main' in dotnet/fsharp/refs/main or 'release/dev17.10' in dotnet/fsharp/refs/release/dev17.10 - name: FSharpReleaseBranchName value: 1espm - # VS Insertion branch name + # VS Insertion branch name (NOT the same as F# branch) # Should be previous release branch in 'main' and 'main' in release branch # (since for all *new* release branches we insert into VS main and for all *previous* releases we insert into corresponding VS release), # i.e. 'rel/d17.9' in dotnet/fsharp/refs/main and 'main' in F# dotnet/fsharp/refs/release/dev17.10 From bb3fc454e807420ba34a7c4c631375e6b1388041 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Tue, 12 Mar 2024 12:58:35 +0100 Subject: [PATCH 26/34] Fixed branch --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d8bba26a76..219239b602 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -88,7 +88,7 @@ extends: # Signed build # #-------------------------------------------------------------------------------------------------------------------# # Localization: we only run it for specific release branches - - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/release/dev17.9') }}: + - ${{ if eq(variables['Build.SourceBranch'], 'refs/heads/release/dev17.10') }}: - template: /eng/common/templates-official/job/onelocbuild.yml@self parameters: MirrorRepo: fsharp From 4479d80c2cc9d42c1760f1ea39b3672c34d0aa70 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Tue, 12 Mar 2024 15:48:33 +0100 Subject: [PATCH 27/34] Adjust VS insertion + log some branch details --- azure-pipelines.yml | 7 ++++--- eng/release/insert-into-vs.yml | 12 ++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 219239b602..b39c9c6308 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -26,13 +26,14 @@ trigger: variables: # Release branch for F# - # Should be 'current' branch name, i.e. 'main' in dotnet/fsharp/refs/main or 'release/dev17.10' in dotnet/fsharp/refs/release/dev17.10 + # Should be 'current' release branch name, i.e. 'release/dev17.10' in dotnet/fsharp/refs/heads/main, 'release/dev17.10' in dotnet/fsharp/refs/heads/release/dev17.10 and 'release/dev17.9' in dotnet/fsharp/refs/heads/release/dev17.9 + # Should **never** be 'main' in dotnet/fsharp/refs/heads/main, since it will start inserting to VS twice. - name: FSharpReleaseBranchName - value: 1espm + value: 1espm #release/dev17.10 # VS Insertion branch name (NOT the same as F# branch) # Should be previous release branch in 'main' and 'main' in release branch # (since for all *new* release branches we insert into VS main and for all *previous* releases we insert into corresponding VS release), - # i.e. 'rel/d17.9' in dotnet/fsharp/refs/main and 'main' in F# dotnet/fsharp/refs/release/dev17.10 + # i.e. 'rel/d17.9' in dotnet/fsharp/refs/heads/main and 'main' in F# dotnet/fsharp/refs/heads/release/dev17.10 - name: VSInsertionTargetBranchName value: main - name: _TeamName diff --git a/eng/release/insert-into-vs.yml b/eng/release/insert-into-vs.yml index e15b691f59..538257079e 100644 --- a/eng/release/insert-into-vs.yml +++ b/eng/release/insert-into-vs.yml @@ -70,6 +70,18 @@ stages: $autoCompleteStr = if ($autoComplete) { 'true' } else { 'false' } Write-Host "Setting InsertAutoComplete to '$autoCompleteStr'" Write-Host "##vso[task.setvariable variable=InsertAutoComplete]$autoCompleteStr" + - task: PowerShell@2 + displayName: Log insertion details + env: + BuildSourceBranch: $(Build.SourceBranch) + inputs: + targetType: 'inline' + pwsh: true + script: | + $componentBranchName = '$ {{ parameters.componentBranchName }}' + $insertBranchName = $env:INSERTTARGETBRANCH + $buildSourceBranch = $env:BUILDSOURCEBRANCH + Write-Host "ComponentBranchName: $componentBranchName; InsertBranchName:$insertBranchName; BuildSourceBranch: $buildSourceBranch" - task: ms-vseng.MicroBuildShipTasks.55100717-a81d-45ea-a363-b8fe3ec375ad.MicroBuildInsertVsPayload@3 displayName: 'Insert VS Payload' inputs: From 407e48faa8fa7943b219ed86ee07b836cf463049 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Tue, 12 Mar 2024 15:58:29 +0100 Subject: [PATCH 28/34] Adjust VS insertion + log some branch details --- azure-pipelines.yml | 4 ++-- eng/release/insert-into-vs.yml | 12 ------------ 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b39c9c6308..a709895458 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -234,8 +234,8 @@ extends: #---------------------------------------------------------------------------------------------------------------------# - template: eng/release/insert-into-vs.yml@self parameters: - componentBranchName: refs/heads/$(FSharpReleaseBranchName) - insertTargetBranch: $(VSInsertionBranchName) + componentBranchName: refs/heads/${{ FSharpReleaseBranchName }} + insertTargetBranch: ${{ VSInsertionBranchName }} insertTeamEmail: fsharpteam@microsoft.com insertTeamName: 'F#' completeInsertion: 'auto' diff --git a/eng/release/insert-into-vs.yml b/eng/release/insert-into-vs.yml index 538257079e..e15b691f59 100644 --- a/eng/release/insert-into-vs.yml +++ b/eng/release/insert-into-vs.yml @@ -70,18 +70,6 @@ stages: $autoCompleteStr = if ($autoComplete) { 'true' } else { 'false' } Write-Host "Setting InsertAutoComplete to '$autoCompleteStr'" Write-Host "##vso[task.setvariable variable=InsertAutoComplete]$autoCompleteStr" - - task: PowerShell@2 - displayName: Log insertion details - env: - BuildSourceBranch: $(Build.SourceBranch) - inputs: - targetType: 'inline' - pwsh: true - script: | - $componentBranchName = '$ {{ parameters.componentBranchName }}' - $insertBranchName = $env:INSERTTARGETBRANCH - $buildSourceBranch = $env:BUILDSOURCEBRANCH - Write-Host "ComponentBranchName: $componentBranchName; InsertBranchName:$insertBranchName; BuildSourceBranch: $buildSourceBranch" - task: ms-vseng.MicroBuildShipTasks.55100717-a81d-45ea-a363-b8fe3ec375ad.MicroBuildInsertVsPayload@3 displayName: 'Insert VS Payload' inputs: From 8ae9125e1a1697cfa4606491a9fc9a89bb1148b9 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Tue, 12 Mar 2024 16:29:40 +0100 Subject: [PATCH 29/34] Fix template variables use --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a709895458..7d91bbf1f3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -234,8 +234,8 @@ extends: #---------------------------------------------------------------------------------------------------------------------# - template: eng/release/insert-into-vs.yml@self parameters: - componentBranchName: refs/heads/${{ FSharpReleaseBranchName }} - insertTargetBranch: ${{ VSInsertionBranchName }} + componentBranchName: refs/heads/${{ varizbles.FSharpReleaseBranchName }} + insertTargetBranch: ${{ vzariables.VSInsertionBranchName }} insertTeamEmail: fsharpteam@microsoft.com insertTeamName: 'F#' completeInsertion: 'auto' From 5fd4820b9a665f524cb379f3d14dd450ea0f02d4 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Tue, 12 Mar 2024 16:30:09 +0100 Subject: [PATCH 30/34] Fix template variables use --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7d91bbf1f3..294977da59 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -234,7 +234,7 @@ extends: #---------------------------------------------------------------------------------------------------------------------# - template: eng/release/insert-into-vs.yml@self parameters: - componentBranchName: refs/heads/${{ varizbles.FSharpReleaseBranchName }} + componentBranchName: refs/heads/${{ vzariables.FSharpReleaseBranchName }} insertTargetBranch: ${{ vzariables.VSInsertionBranchName }} insertTeamEmail: fsharpteam@microsoft.com insertTeamName: 'F#' From 2a03c25b0e47186ddcdcf237d7c7271e4b116efb Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Tue, 12 Mar 2024 16:31:19 +0100 Subject: [PATCH 31/34] Fix template variables use --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 294977da59..09447d7998 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -234,8 +234,8 @@ extends: #---------------------------------------------------------------------------------------------------------------------# - template: eng/release/insert-into-vs.yml@self parameters: - componentBranchName: refs/heads/${{ vzariables.FSharpReleaseBranchName }} - insertTargetBranch: ${{ vzariables.VSInsertionBranchName }} + componentBranchName: refs/heads/${{ variables.FSharpReleaseBranchName }} + insertTargetBranch: ${{ variables.VSInsertionBranchName }} insertTeamEmail: fsharpteam@microsoft.com insertTeamName: 'F#' completeInsertion: 'auto' From aec2a4751e8e2efeac09474f0aabd9ac4cad5ed9 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Tue, 12 Mar 2024 18:32:27 +0100 Subject: [PATCH 32/34] Fix VS target branch name --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 09447d7998..8ffbd71c95 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -235,7 +235,7 @@ extends: - template: eng/release/insert-into-vs.yml@self parameters: componentBranchName: refs/heads/${{ variables.FSharpReleaseBranchName }} - insertTargetBranch: ${{ variables.VSInsertionBranchName }} + insertTargetBranch: ${{ variables.VSInsertionTargetBranchName }} insertTeamEmail: fsharpteam@microsoft.com insertTeamName: 'F#' completeInsertion: 'auto' From a0081443628b0c582abe66f83944519378d2a5dd Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Fri, 15 Mar 2024 16:55:26 +0100 Subject: [PATCH 33/34] Enable autobaseline --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 8ffbd71c95..bf85cf4f85 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -67,8 +67,8 @@ resources: extends: template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate parameters: - #featureFlags: - # autoBaseline: true + featureFlags: + autoBaseline: true sdl: binskim: enabled: true @@ -76,7 +76,7 @@ extends: enabled: true sbom: enabled: false # VS SBOM is generated with other steps - justificationForDisabling: 'SBOM for F# is generated via build process.' + justificationForDisabling: 'SBOM for F# is generated via build process. Will be migrated at later date.' sourceAnalysisPool: name: NetCore1ESPool-Svc-Internal image: 1es-windows-2022-pt From 58a6bb63b504e8b1af140700681af2de31237062 Mon Sep 17 00:00:00 2001 From: Vlad Zarytovskii Date: Mon, 18 Mar 2024 11:19:25 +0100 Subject: [PATCH 34/34] Use correct branch name --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bf85cf4f85..e6535fbc1a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -29,7 +29,7 @@ variables: # Should be 'current' release branch name, i.e. 'release/dev17.10' in dotnet/fsharp/refs/heads/main, 'release/dev17.10' in dotnet/fsharp/refs/heads/release/dev17.10 and 'release/dev17.9' in dotnet/fsharp/refs/heads/release/dev17.9 # Should **never** be 'main' in dotnet/fsharp/refs/heads/main, since it will start inserting to VS twice. - name: FSharpReleaseBranchName - value: 1espm #release/dev17.10 + value: release/dev17.10 # VS Insertion branch name (NOT the same as F# branch) # Should be previous release branch in 'main' and 'main' in release branch # (since for all *new* release branches we insert into VS main and for all *previous* releases we insert into corresponding VS release),