From 210accf5bf97abbdab42d72af1a3cafbed6b33c3 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 31 Jul 2025 14:39:10 +0000 Subject: [PATCH 1/9] Initial plan From 008d3e2aabd65ff7366e792c316aef6f73f818c5 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 31 Jul 2025 14:56:55 +0000 Subject: [PATCH 2/9] Add Azure DevOps regression testing pipeline for F# compiler Co-authored-by: T-Gro <46543583+T-Gro@users.noreply.github.com> --- azure-pipelines-regression-test.yml | 199 ++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 azure-pipelines-regression-test.yml diff --git a/azure-pipelines-regression-test.yml b/azure-pipelines-regression-test.yml new file mode 100644 index 00000000000..8c020cc79e8 --- /dev/null +++ b/azure-pipelines-regression-test.yml @@ -0,0 +1,199 @@ +# Regression Testing Pipeline +# Tests F# compiler regressions by building third-party F# projects with the freshly built compiler + +trigger: + branches: + include: + - main + - release/* + - feature/* + paths: + include: + - src/Compiler/* + - src/fsc/* + - src/FSharp.Core/* + - src/FSharp.Build/* + exclude: + - docs/* + - .github/* + - '*.md' + +pr: + branches: + include: + - main + - release/* + - feature/* + paths: + include: + - src/Compiler/* + - src/fsc/* + - src/FSharp.Core/* + - src/FSharp.Build/* + exclude: + - docs/* + - .github/* + - '*.md' + +variables: + - name: _TeamName + value: FSharp + - name: _BuildConfig + value: Release + - name: _PublishUsingPipelines + value: true + - name: Codeql.Enabled + value: false # Disabled for regression tests + # Pick up pool provider name behavior from shared yaml template + - template: /eng/common/templates/variables/pool-providers.yml + +# Matrix of third-party repositories to test against +# This configuration can be extended in the future +parameters: +- name: testMatrix + type: object + default: + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.cmd + displayName: FSharpPlus + +stages: +- stage: RegressionTest + displayName: F# Compiler Regression Tests + jobs: + - job: BuildCompiler + displayName: Build F# Compiler + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64.open + timeoutInMinutes: 60 + steps: + - checkout: self + clean: true + displayName: Checkout F# compiler source + + - script: dotnet --list-sdks + displayName: Report .NET SDK versions + + - script: .\Build.cmd -c $(_BuildConfig) -pack + env: + NativeToolsOnMachine: true + displayName: Build F# compiler and create packages + + - task: PublishPipelineArtifact@1 + displayName: Publish F# Compiler Artifacts + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts' + artifactName: 'FSharpCompilerArtifacts' + publishLocation: pipeline + condition: succeeded() + + - task: PublishPipelineArtifact@1 + displayName: Publish UseLocalCompiler props file + inputs: + targetPath: '$(Build.SourcesDirectory)/UseLocalCompiler.Directory.Build.props' + artifactName: 'UseLocalCompilerProps' + publishLocation: pipeline + condition: succeeded() + + # Test against third-party repositories + - ${{ each item in parameters.testMatrix }}: + - job: Test_${{ replace(item.repo, '/', '_') }} + displayName: 'Regression Test: ${{ item.displayName }}' + dependsOn: BuildCompiler + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals windows.vs2022.amd64.open + timeoutInMinutes: 60 + variables: + TestRepoName: ${{ item.repo }} + TestCommit: ${{ item.commit }} + BuildScript: ${{ item.buildScript }} + DisplayName: ${{ item.displayName }} + steps: + - checkout: none + displayName: Skip default checkout + + # Download the F# compiler artifacts + - task: DownloadPipelineArtifact@2 + displayName: Download F# Compiler Artifacts + inputs: + artifactName: 'FSharpCompilerArtifacts' + downloadPath: '$(Pipeline.Workspace)/FSharpCompiler' + + - task: DownloadPipelineArtifact@2 + displayName: Download UseLocalCompiler props + inputs: + artifactName: 'UseLocalCompilerProps' + downloadPath: '$(Pipeline.Workspace)/Props' + + # Checkout the third-party repository at specific commit + - script: | + git clone https://github.com/$(TestRepoName).git $(Pipeline.Workspace)/TestRepo + cd $(Pipeline.Workspace)/TestRepo + git checkout $(TestCommit) + echo "Checked out $(TestRepoName) at commit $(TestCommit)" + git log -1 --oneline + displayName: 'Checkout $(DisplayName) at specific commit' + + # Inject UseLocalCompiler.Directory.Build.props + - script: | + cd $(Pipeline.Workspace)/TestRepo + echo Creating Directory.Build.props to use local F# compiler + echo ^ > Directory.Build.props + echo ^ >> Directory.Build.props + echo ^ >> Directory.Build.props + + echo Copying UseLocalCompiler.Directory.Build.props with updated path + powershell -Command "& { $content = Get-Content '$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props' -Raw; $content = $content -replace 'LocalFSharpCompilerPath.*MSBuildThisFileDirectory.*', 'LocalFSharpCompilerPath>$(Pipeline.Workspace)/FSharpCompiler<'; Set-Content 'UseLocalCompiler.Directory.Build.props' -Value $content }" + + echo Directory.Build.props content: + type Directory.Build.props + echo. + echo UseLocalCompiler.Directory.Build.props content (first 20 lines): + powershell -Command "Get-Content 'UseLocalCompiler.Directory.Build.props' | Select-Object -First 20" + displayName: 'Setup local compiler configuration for $(DisplayName)' + + # Report dotnet info in test environment + - script: | + cd $(Pipeline.Workspace)/TestRepo + dotnet --info + echo. + echo F# Compiler artifacts available: + dir "$(Pipeline.Workspace)\FSharpCompiler\bin\fsc\Release\net9.0" /B + displayName: 'Report build environment for $(DisplayName)' + + # Build the third-party project using local F# compiler + - script: | + cd $(Pipeline.Workspace)/TestRepo + echo Running build script: $(BuildScript) + call $(BuildScript) + displayName: 'Build $(DisplayName) with local F# compiler' + env: + # Ensure the build uses local compiler + LoadLocalFSharpBuild: true + LocalFSharpCompilerConfiguration: Release + timeoutInMinutes: 45 + + # Capture and publish build logs on failure + - task: PublishPipelineArtifact@1 + displayName: 'Publish $(DisplayName) Build Logs' + inputs: + targetPath: '$(Pipeline.Workspace)/TestRepo' + artifactName: '$(DisplayName)_BuildOutput' + publishLocation: pipeline + condition: always() + continueOnError: true + + # Report success/failure + - script: | + echo. + echo ============================================ + echo Regression test completed for $(DisplayName) + echo Repository: $(TestRepoName) + echo Commit: $(TestCommit) + echo Status: %AGENT_JOBSTATUS% + echo ============================================ + displayName: 'Report $(DisplayName) test result' + condition: always() \ No newline at end of file From a268b0fa8220287407d63c5c9ddad5deddbd0161 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 31 Jul 2025 15:01:44 +0000 Subject: [PATCH 3/9] Complete regression testing pipeline with documentation and improvements Co-authored-by: T-Gro <46543583+T-Gro@users.noreply.github.com> --- azure-pipelines-regression-test.yml | 120 +++++++++++++++++++- docs/regression-testing-pipeline.md | 166 ++++++++++++++++++++++++++++ 2 files changed, 281 insertions(+), 5 deletions(-) create mode 100644 docs/regression-testing-pipeline.md diff --git a/azure-pipelines-regression-test.yml b/azure-pipelines-regression-test.yml index 8c020cc79e8..0b397b2883a 100644 --- a/azure-pipelines-regression-test.yml +++ b/azure-pipelines-regression-test.yml @@ -1,5 +1,23 @@ -# Regression Testing Pipeline -# Tests F# compiler regressions by building third-party F# projects with the freshly built compiler +# Regression Testing Pipeline for F# Compiler +# +# This pipeline tests for regressions in the F# compiler by building third-party F# projects +# with the freshly built compiler from this repository. +# +# Purpose: +# - Catch regressions early by testing the compiler against real-world F# libraries +# - Provide detailed build logs when third-party builds fail with the new compiler +# - Support extensible matrix configuration for testing multiple libraries +# +# Matrix Configuration: +# The testMatrix parameter supports multiple repositories with their specific build requirements. +# Each entry should specify: +# - repo: GitHub repository in format "owner/name" +# - commit: Specific commit SHA to test against for reproducible results +# - buildScript: Build script to execute (relative to repo root) +# - displayName: Human-readable name for the test job +# +# Current matrix includes: +# - FSharpPlus: A popular F# library with extensive use of advanced F# features trigger: branches: @@ -76,6 +94,20 @@ stages: - script: dotnet --list-sdks displayName: Report .NET SDK versions + # We first download a publicly available .NET SDK. That one has support for `path` in global.json. + # dotnet.cmd script can then download a version which is not yet shipped, but matches global.json. + - task: UseDotNet@2 + displayName: install SDK + inputs: + packageType: sdk + version: '10.x' + includePreviewVersions: true + workingDirectory: $(Build.SourcesDirectory) + installationPath: $(Build.SourcesDirectory)/.dotnet + + - script: .\eng\common\dotnet.cmd + displayName: Ensure correct .NET SDK version + - script: .\Build.cmd -c $(_BuildConfig) -pack env: NativeToolsOnMachine: true @@ -130,11 +162,28 @@ stages: # Checkout the third-party repository at specific commit - script: | + echo Cloning repository: $(TestRepoName) git clone https://github.com/$(TestRepoName).git $(Pipeline.Workspace)/TestRepo cd $(Pipeline.Workspace)/TestRepo + + echo Checking out commit: $(TestCommit) git checkout $(TestCommit) - echo "Checked out $(TestRepoName) at commit $(TestCommit)" + + echo Successfully checked out $(TestRepoName) at commit $(TestCommit) git log -1 --oneline + + echo Repository structure: + dir /B + + echo Verifying build script exists: $(BuildScript) + if exist "$(BuildScript)" ( + echo ✓ Build script found: $(BuildScript) + ) else ( + echo ✗ Build script not found: $(BuildScript) + echo Available files in root: + dir + exit /b 1 + ) displayName: 'Checkout $(DisplayName) at specific commit' # Inject UseLocalCompiler.Directory.Build.props @@ -158,17 +207,49 @@ stages: # Report dotnet info in test environment - script: | cd $(Pipeline.Workspace)/TestRepo + echo =========================================== + echo Environment Information for $(DisplayName) + echo =========================================== dotnet --info echo. + echo MSBuild version: + dotnet msbuild -version + echo. echo F# Compiler artifacts available: dir "$(Pipeline.Workspace)\FSharpCompiler\bin\fsc\Release\net9.0" /B + echo. + echo F# Core available: + if exist "$(Pipeline.Workspace)\FSharpCompiler\bin\FSharp.Core\Release\netstandard2.0\FSharp.Core.dll" ( + echo ✓ FSharp.Core.dll found + ) else ( + echo ✗ FSharp.Core.dll not found + ) + echo. + echo Directory.Build.props content: + type Directory.Build.props + echo. + echo =========================================== displayName: 'Report build environment for $(DisplayName)' # Build the third-party project using local F# compiler - script: | cd $(Pipeline.Workspace)/TestRepo - echo Running build script: $(BuildScript) + echo ============================================ + echo Starting build for $(DisplayName) + echo Repository: $(TestRepoName) + echo Commit: $(TestCommit) + echo Build Script: $(BuildScript) + echo ============================================ + echo. + + echo Executing: $(BuildScript) call $(BuildScript) + + echo. + echo ============================================ + echo Build completed for $(DisplayName) + echo Exit code: %ERRORLEVEL% + echo ============================================ displayName: 'Build $(DisplayName) with local F# compiler' env: # Ensure the build uses local compiler @@ -186,14 +267,43 @@ stages: condition: always() continueOnError: true + # Publish any MSBuild binary logs if they exist + - task: PublishPipelineArtifact@1 + displayName: 'Publish $(DisplayName) Binary Logs' + inputs: + targetPath: '$(Pipeline.Workspace)/TestRepo' + artifactName: '$(DisplayName)_BinaryLogs' + publishLocation: pipeline + condition: always() + continueOnError: true + # Report success/failure - script: | + cd $(Pipeline.Workspace)/TestRepo echo. echo ============================================ echo Regression test completed for $(DisplayName) echo Repository: $(TestRepoName) echo Commit: $(TestCommit) - echo Status: %AGENT_JOBSTATUS% + echo Build Script: $(BuildScript) + if "%AGENT_JOBSTATUS%"=="Succeeded" ( + echo Status: ✓ SUCCESS + echo The $(DisplayName) library builds successfully with the new F# compiler + ) else ( + echo Status: ✗ FAILED + echo The $(DisplayName) library failed to build with the new F# compiler + echo Check the build logs and artifacts for details + ) echo ============================================ + + echo Build artifacts in repository: + if exist "bin" ( + echo Checking bin directory: + dir bin /S /B 2>nul | findstr /R "^.*$" | findstr /N "^" | findstr "^[1-9]:" + ) + if exist "*.binlog" ( + echo Binary logs found: + dir *.binlog /B 2>nul + ) displayName: 'Report $(DisplayName) test result' condition: always() \ No newline at end of file diff --git a/docs/regression-testing-pipeline.md b/docs/regression-testing-pipeline.md new file mode 100644 index 00000000000..d0fe722d91d --- /dev/null +++ b/docs/regression-testing-pipeline.md @@ -0,0 +1,166 @@ +# F# Compiler Regression Testing Pipeline + +This document describes the F# compiler regression testing pipeline implemented in `azure-pipelines-regression-test.yml`. + +## Purpose + +The regression testing pipeline helps catch F# compiler regressions by building popular third-party F# libraries with the freshly built compiler from this repository. This provides early detection of breaking changes that might affect real-world F# projects. + +## How It Works + +### Pipeline Workflow + +1. **Build F# Compiler**: The pipeline first builds the F# compiler and packages from the current source code using the standard build process. + +2. **Test Against Third-Party Libraries**: For each library in the test matrix: + - Checkout the third-party repository at a specific commit + - Inject the `UseLocalCompiler.Directory.Build.props` configuration to use the locally built compiler + - Run the library's build script + - Capture detailed build logs and artifacts + +3. **Report Results**: Success/failure status is reported with detailed logs for diagnosis. + +### Key Features + +- **Reproducible Testing**: Uses specific commit SHAs for third-party libraries to ensure consistent results +- **Matrix Configuration**: Supports testing multiple libraries with different build requirements +- **Detailed Logging**: Captures comprehensive build logs, binary logs, and environment information +- **Artifact Publishing**: Publishes build outputs for analysis when builds fail + +## Current Test Matrix + +The pipeline currently tests against: + +| Library | Repository | Commit | Build Script | Purpose | +|---------|------------|--------|--------------|---------| +| FSharpPlus | fsprojects/FSharpPlus | f614035b75922aba41ed6a36c2fc986a2171d2b8 | build.cmd | Tests advanced F# language features | + +## Adding New Libraries + +To add a new library to the test matrix: + +1. **Choose a Library**: Select a representative F# library that uses features you want to test. + +2. **Find a Stable Commit**: Choose a specific commit SHA that is known to build successfully with the current F# compiler. + +3. **Update the Matrix**: Add an entry to the `testMatrix` parameter in the pipeline: + +```yaml +parameters: +- name: testMatrix + type: object + default: + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.cmd + displayName: FSharpPlus + - repo: your-org/your-library # Add your library here + commit: abc123def456... # Specific commit SHA + buildScript: build.sh # Build script (build.cmd, build.sh, etc.) + displayName: YourLibrary # Human-readable name +``` + +4. **Test the Configuration**: Verify that your library builds correctly with the current compiler before adding it to the matrix. + +## Pipeline Configuration + +### Triggers + +The pipeline is triggered by: +- **Branches**: main, release/*, feature/* +- **Paths**: Changes to compiler source code (src/Compiler/, src/fsc/, src/FSharp.Core/, src/FSharp.Build/) +- **Exclusions**: Documentation and non-compiler changes + +### Build Environment + +- **OS**: Windows (windows.vs2022.amd64.open) +- **Pool**: Uses the standard public build pool (`DncEngPublicBuildPool`) +- **Timeout**: 60 minutes per job +- **.NET SDK**: Automatically installs the required preview SDK + +### Artifacts + +The pipeline publishes several artifacts for analysis: +- **FSharpCompilerArtifacts**: Complete F# compiler build output +- **UseLocalCompilerProps**: Configuration file for using local compiler +- **{Library}_BuildOutput**: Complete build output from each tested library +- **{Library}_BinaryLogs**: MSBuild binary logs when available + +## Troubleshooting Build Failures + +When a regression test fails: + +1. **Check the Job Summary**: Look at the final status report for high-level information. + +2. **Download Build Logs**: Download the published artifacts to examine detailed build output. + +3. **Compare Compiler Changes**: Review what changes were made to the compiler that might affect the failing library. + +4. **Local Reproduction**: Use the `UseLocalCompiler.Directory.Build.props` file to reproduce the issue locally. + +### Local Testing + +To test a library locally with your F# compiler build: + +1. Build the F# compiler: `.\Build.cmd -c Release -pack` + +2. In the third-party library directory, create a `Directory.Build.props`: + ```xml + + + + ``` + +3. Update the `LocalFSharpCompilerPath` in `UseLocalCompiler.Directory.Build.props` to point to your F# repository. + +4. Set environment variables: + ```cmd + set LoadLocalFSharpBuild=true + set LocalFSharpCompilerConfiguration=Release + ``` + +5. Run the library's build script. + +## Best Practices + +### For Library Selection + +- **Coverage**: Choose libraries that exercise different F# language features +- **Popularity**: Include widely-used libraries that represent real-world usage +- **Stability**: Use libraries with stable build processes and minimal external dependencies +- **Diversity**: Include libraries with different build systems and target frameworks + +### For Maintenance + +- **Regular Updates**: Periodically update commit SHAs to newer stable versions +- **Monitor Dependencies**: Watch for changes in third-party library build requirements +- **Baseline Management**: Update baselines when intentional breaking changes are made + +## Technical Details + +### UseLocalCompiler.Directory.Build.props + +This MSBuild props file configures projects to use the locally built F# compiler instead of the SDK version. Key settings: + +- `LocalFSharpCompilerPath`: Points to the F# compiler artifacts +- `DotnetFscCompilerPath`: Path to the fsc.dll compiler +- `DisableImplicitFSharpCoreReference`: Ensures local FSharp.Core is used + +### Path Handling + +The pipeline dynamically updates paths in the props file using PowerShell: +```powershell +$content -replace 'LocalFSharpCompilerPath.*MSBuildThisFileDirectory.*', 'LocalFSharpCompilerPath>$(Pipeline.Workspace)/FSharpCompiler<' +``` + +This ensures the correct path is used in the Azure DevOps environment. + +## Future Enhancements + +Potential improvements to the pipeline: + +1. **Performance Testing**: Measure compilation times and memory usage +2. **Multiple Target Frameworks**: Test libraries across different .NET versions +3. **Parallel Execution**: Run library tests in parallel for faster feedback +4. **Automatic Bisection**: Automatically identify which commit introduced a regression +5. **Integration with GitHub**: Post regression test results as PR comments \ No newline at end of file From 1f17e6b8e7ef4b36e7785dcb246eaeba49552eed Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 Aug 2025 09:09:16 +0000 Subject: [PATCH 4/9] Address feedback on Azure DevOps regression testing pipeline - Simplify trigger paths to use 'src/*' instead of specific subdirectories - Update pool configuration to use $(WindowsMachineQueueName) variable - Replace cmd/batch scripts with PowerShell tasks for better error handling - Add F# script-based Directory.Build.props setup logic to properly handle existing files - Add .NET SDK version detection from global.json in test repositories - Use LocalFSharpCompilerPath environment variable instead of modifying props files - Enable MSBuild binary logging via environment variables - Focus artifact publishing on binary logs only instead of entire repository output - Improve error reporting and environment information display Co-authored-by: T-Gro <46543583+T-Gro@users.noreply.github.com> --- azure-pipelines-regression-test.yml | 349 ++++++++++++++++------------ 1 file changed, 194 insertions(+), 155 deletions(-) diff --git a/azure-pipelines-regression-test.yml b/azure-pipelines-regression-test.yml index 0b397b2883a..754a9ea8f4e 100644 --- a/azure-pipelines-regression-test.yml +++ b/azure-pipelines-regression-test.yml @@ -1,23 +1,5 @@ -# Regression Testing Pipeline for F# Compiler -# -# This pipeline tests for regressions in the F# compiler by building third-party F# projects -# with the freshly built compiler from this repository. -# -# Purpose: -# - Catch regressions early by testing the compiler against real-world F# libraries -# - Provide detailed build logs when third-party builds fail with the new compiler -# - Support extensible matrix configuration for testing multiple libraries -# -# Matrix Configuration: -# The testMatrix parameter supports multiple repositories with their specific build requirements. -# Each entry should specify: -# - repo: GitHub repository in format "owner/name" -# - commit: Specific commit SHA to test against for reproducible results -# - buildScript: Build script to execute (relative to repo root) -# - displayName: Human-readable name for the test job -# -# Current matrix includes: -# - FSharpPlus: A popular F# library with extensive use of advanced F# features +# Regression Testing Pipeline +# Tests F# compiler regressions by building third-party F# projects with the freshly built compiler trigger: branches: @@ -27,10 +9,7 @@ trigger: - feature/* paths: include: - - src/Compiler/* - - src/fsc/* - - src/FSharp.Core/* - - src/FSharp.Build/* + - src/* exclude: - docs/* - .github/* @@ -44,10 +23,7 @@ pr: - feature/* paths: include: - - src/Compiler/* - - src/fsc/* - - src/FSharp.Core/* - - src/FSharp.Build/* + - src/* exclude: - docs/* - .github/* @@ -84,18 +60,13 @@ stages: displayName: Build F# Compiler pool: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64.open + demands: ImageOverride -equals $(WindowsMachineQueueName) timeoutInMinutes: 60 steps: - checkout: self clean: true displayName: Checkout F# compiler source - - script: dotnet --list-sdks - displayName: Report .NET SDK versions - - # We first download a publicly available .NET SDK. That one has support for `path` in global.json. - # dotnet.cmd script can then download a version which is not yet shipped, but matches global.json. - task: UseDotNet@2 displayName: install SDK inputs: @@ -136,7 +107,7 @@ stages: dependsOn: BuildCompiler pool: name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64.open + demands: ImageOverride -equals $(WindowsMachineQueueName) timeoutInMinutes: 60 variables: TestRepoName: ${{ item.repo }} @@ -161,113 +132,186 @@ stages: downloadPath: '$(Pipeline.Workspace)/Props' # Checkout the third-party repository at specific commit - - script: | - echo Cloning repository: $(TestRepoName) - git clone https://github.com/$(TestRepoName).git $(Pipeline.Workspace)/TestRepo - cd $(Pipeline.Workspace)/TestRepo - - echo Checking out commit: $(TestCommit) - git checkout $(TestCommit) - - echo Successfully checked out $(TestRepoName) at commit $(TestCommit) - git log -1 --oneline - - echo Repository structure: - dir /B - - echo Verifying build script exists: $(BuildScript) - if exist "$(BuildScript)" ( - echo ✓ Build script found: $(BuildScript) - ) else ( - echo ✗ Build script not found: $(BuildScript) - echo Available files in root: - dir - exit /b 1 - ) + - task: PowerShell@2 displayName: 'Checkout $(DisplayName) at specific commit' + inputs: + script: | + Write-Host "Cloning repository: $(TestRepoName)" + git clone https://github.com/$(TestRepoName).git $(Pipeline.Workspace)/TestRepo + Set-Location $(Pipeline.Workspace)/TestRepo + + Write-Host "Checking out commit: $(TestCommit)" + git checkout $(TestCommit) + + Write-Host "Successfully checked out $(TestRepoName) at commit $(TestCommit)" + git log -1 --oneline + + Write-Host "Repository structure:" + Get-ChildItem -Name + + Write-Host "Verifying build script exists: $(BuildScript)" + if (Test-Path "$(BuildScript)") { + Write-Host "✓ Build script found: $(BuildScript)" + } else { + Write-Host "✗ Build script not found: $(BuildScript)" + Write-Host "Available files in root:" + Get-ChildItem + exit 1 + } - # Inject UseLocalCompiler.Directory.Build.props - - script: | - cd $(Pipeline.Workspace)/TestRepo - echo Creating Directory.Build.props to use local F# compiler - echo ^ > Directory.Build.props - echo ^ >> Directory.Build.props - echo ^ >> Directory.Build.props - - echo Copying UseLocalCompiler.Directory.Build.props with updated path - powershell -Command "& { $content = Get-Content '$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props' -Raw; $content = $content -replace 'LocalFSharpCompilerPath.*MSBuildThisFileDirectory.*', 'LocalFSharpCompilerPath>$(Pipeline.Workspace)/FSharpCompiler<'; Set-Content 'UseLocalCompiler.Directory.Build.props' -Value $content }" - - echo Directory.Build.props content: - type Directory.Build.props - echo. - echo UseLocalCompiler.Directory.Build.props content (first 20 lines): - powershell -Command "Get-Content 'UseLocalCompiler.Directory.Build.props' | Select-Object -First 20" + # Detect .NET SDK version from global.json if it exists + - task: PowerShell@2 + displayName: 'Detect .NET SDK version for $(DisplayName)' + inputs: + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + + if (Test-Path "global.json") { + Write-Host "Found global.json, reading .NET SDK version" + $globalJson = Get-Content "global.json" | ConvertFrom-Json + $sdkVersion = $globalJson.sdk.version + Write-Host "Detected .NET SDK version: $sdkVersion" + Write-Host "##vso[task.setvariable variable=DotNetSdkVersion]$sdkVersion" + } else { + Write-Host "No global.json found, will use latest public SDK" + Write-Host "##vso[task.setvariable variable=DotNetSdkVersion]10.x" + } + + # Install appropriate .NET SDK version + - task: UseDotNet@2 + displayName: 'Install .NET SDK for $(DisplayName)' + inputs: + packageType: sdk + version: $(DotNetSdkVersion) + includePreviewVersions: true + workingDirectory: $(Pipeline.Workspace)/TestRepo + installationPath: $(Pipeline.Workspace)/TestRepo/.dotnet + + # Setup Directory.Build.props and UseLocalCompiler configuration using F# script + - task: PowerShell@2 displayName: 'Setup local compiler configuration for $(DisplayName)' + inputs: + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + + # Create F# script to handle Directory.Build.props setup + $fsharpScript = @' + #r "nuget: System.Xml.ReaderWriter" + open System.IO + open System.Xml + + let useLocalCompilerImport = """""" + + let directoryBuildPropsPath = "Directory.Build.props" + + if File.Exists(directoryBuildPropsPath) then + printfn "Directory.Build.props exists, modifying it" + let doc = XmlDocument() + doc.Load(directoryBuildPropsPath) + + // Find the Project element + let projectElement = doc.SelectSingleNode("/Project") + if projectElement <> null then + // Check if our import already exists + let existingImport = doc.SelectSingleNode(sprintf "//Import[@Project='UseLocalCompiler.Directory.Build.props']") + if existingImport = null then + let importElement = doc.CreateElement("Import") + importElement.SetAttribute("Project", "UseLocalCompiler.Directory.Build.props") + projectElement.InsertBefore(importElement, projectElement.FirstChild) |> ignore + doc.Save(directoryBuildPropsPath) + printfn "Added UseLocalCompiler import to existing Directory.Build.props" + else + printfn "UseLocalCompiler import already exists" + else + printfn "Warning: Could not find Project element in Directory.Build.props" + else + printfn "Creating new Directory.Build.props" + let content = sprintf "\n %s\n" useLocalCompilerImport + File.WriteAllText(directoryBuildPropsPath, content) + + printfn "Directory.Build.props content:" + File.ReadAllText(directoryBuildPropsPath) |> printfn "%s" + '@ + + $fsharpScript | Out-File -FilePath "PrepareRepoForTesting.fsx" -Encoding UTF8 + + # Run the F# script using dotnet fsi + dotnet fsi PrepareRepoForTesting.fsx + + # Copy UseLocalCompiler.Directory.Build.props as-is (path will be set via environment variable) + Copy-Item "$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props" "UseLocalCompiler.Directory.Build.props" -Force + + Write-Host "UseLocalCompiler.Directory.Build.props content (first 20 lines):" + Get-Content "UseLocalCompiler.Directory.Build.props" | Select-Object -First 20 # Report dotnet info in test environment - - script: | - cd $(Pipeline.Workspace)/TestRepo - echo =========================================== - echo Environment Information for $(DisplayName) - echo =========================================== - dotnet --info - echo. - echo MSBuild version: - dotnet msbuild -version - echo. - echo F# Compiler artifacts available: - dir "$(Pipeline.Workspace)\FSharpCompiler\bin\fsc\Release\net9.0" /B - echo. - echo F# Core available: - if exist "$(Pipeline.Workspace)\FSharpCompiler\bin\FSharp.Core\Release\netstandard2.0\FSharp.Core.dll" ( - echo ✓ FSharp.Core.dll found - ) else ( - echo ✗ FSharp.Core.dll not found - ) - echo. - echo Directory.Build.props content: - type Directory.Build.props - echo. - echo =========================================== + - task: PowerShell@2 displayName: 'Report build environment for $(DisplayName)' + inputs: + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "===========================================" + Write-Host "Environment Information for $(DisplayName)" + Write-Host "===========================================" + dotnet --info + Write-Host "" + Write-Host "MSBuild version:" + dotnet msbuild -version + Write-Host "" + Write-Host "F# Compiler artifacts available:" + Get-ChildItem "$(Pipeline.Workspace)\FSharpCompiler\bin\fsc\Release\net9.0" -Name + Write-Host "" + Write-Host "F# Core available:" + if (Test-Path "$(Pipeline.Workspace)\FSharpCompiler\bin\FSharp.Core\Release\netstandard2.0\FSharp.Core.dll") { + Write-Host "✓ FSharp.Core.dll found" + } else { + Write-Host "✗ FSharp.Core.dll not found" + } + Write-Host "" + Write-Host "Directory.Build.props content:" + Get-Content "Directory.Build.props" + Write-Host "" + Write-Host "===========================================" # Build the third-party project using local F# compiler - - script: | - cd $(Pipeline.Workspace)/TestRepo - echo ============================================ - echo Starting build for $(DisplayName) - echo Repository: $(TestRepoName) - echo Commit: $(TestCommit) - echo Build Script: $(BuildScript) - echo ============================================ - echo. - - echo Executing: $(BuildScript) - call $(BuildScript) - - echo. - echo ============================================ - echo Build completed for $(DisplayName) - echo Exit code: %ERRORLEVEL% - echo ============================================ + - task: PowerShell@2 displayName: 'Build $(DisplayName) with local F# compiler' env: - # Ensure the build uses local compiler + # Set environment variables to use local compiler and enforce binary logs + LocalFSharpCompilerPath: $(Pipeline.Workspace)/FSharpCompiler LoadLocalFSharpBuild: true LocalFSharpCompilerConfiguration: Release + # Force MSBuild binary logs + MSBUILDBINARYLOGGERENABLED: true + MSBUILDBINARYLOGGER: "*.binlog" timeoutInMinutes: 45 - - # Capture and publish build logs on failure - - task: PublishPipelineArtifact@1 - displayName: 'Publish $(DisplayName) Build Logs' inputs: - targetPath: '$(Pipeline.Workspace)/TestRepo' - artifactName: '$(DisplayName)_BuildOutput' - publishLocation: pipeline - condition: always() - continueOnError: true + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "============================================" + Write-Host "Starting build for $(DisplayName)" + Write-Host "Repository: $(TestRepoName)" + Write-Host "Commit: $(TestCommit)" + Write-Host "Build Script: $(BuildScript)" + Write-Host "============================================" + Write-Host "" + + Write-Host "Executing: $(BuildScript)" + cmd /c "$(BuildScript)" + $exitCode = $LASTEXITCODE + + Write-Host "" + Write-Host "============================================" + Write-Host "Build completed for $(DisplayName)" + Write-Host "Exit code: $exitCode" + Write-Host "============================================" + + if ($exitCode -ne 0) { + exit $exitCode + } - # Publish any MSBuild binary logs if they exist + # Capture and publish build logs on failure + # Publish only MSBuild binary logs - task: PublishPipelineArtifact@1 displayName: 'Publish $(DisplayName) Binary Logs' inputs: @@ -278,32 +322,27 @@ stages: continueOnError: true # Report success/failure - - script: | - cd $(Pipeline.Workspace)/TestRepo - echo. - echo ============================================ - echo Regression test completed for $(DisplayName) - echo Repository: $(TestRepoName) - echo Commit: $(TestCommit) - echo Build Script: $(BuildScript) - if "%AGENT_JOBSTATUS%"=="Succeeded" ( - echo Status: ✓ SUCCESS - echo The $(DisplayName) library builds successfully with the new F# compiler - ) else ( - echo Status: ✗ FAILED - echo The $(DisplayName) library failed to build with the new F# compiler - echo Check the build logs and artifacts for details - ) - echo ============================================ - - echo Build artifacts in repository: - if exist "bin" ( - echo Checking bin directory: - dir bin /S /B 2>nul | findstr /R "^.*$" | findstr /N "^" | findstr "^[1-9]:" - ) - if exist "*.binlog" ( - echo Binary logs found: - dir *.binlog /B 2>nul - ) + - task: PowerShell@2 displayName: 'Report $(DisplayName) test result' - condition: always() \ No newline at end of file + condition: always() + inputs: + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "" + Write-Host "============================================" + Write-Host "Regression test completed for $(DisplayName)" + Write-Host "Repository: $(TestRepoName)" + Write-Host "Commit: $(TestCommit)" + Write-Host "Build Script: $(BuildScript)" + if ($env:AGENT_JOBSTATUS -eq "Succeeded") { + Write-Host "Status: ✓ SUCCESS" + Write-Host "The $(DisplayName) library builds successfully with the new F# compiler" + } else { + Write-Host "Status: ✗ FAILED" + Write-Host "The $(DisplayName) library failed to build with the new F# compiler" + Write-Host "Check the build logs and artifacts for details" + } + Write-Host "============================================" + + Write-Host "Binary logs found:" + Get-ChildItem "*.binlog" -ErrorAction SilentlyContinue | ForEach-Object { Write-Host $_.Name } \ No newline at end of file From a19160db28cbcf883df409f497158600c1a48583 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 Aug 2025 10:06:24 +0000 Subject: [PATCH 5/9] Address technical feedback - simplify SDK detection, fix file paths, focus on binlog artifacts Co-authored-by: T-Gro <46543583+T-Gro@users.noreply.github.com> --- azure-pipelines-regression-test.yml | 45 ++++++++--------------------- 1 file changed, 12 insertions(+), 33 deletions(-) diff --git a/azure-pipelines-regression-test.yml b/azure-pipelines-regression-test.yml index 754a9ea8f4e..f7c85634045 100644 --- a/azure-pipelines-regression-test.yml +++ b/azure-pipelines-regression-test.yml @@ -56,8 +56,10 @@ stages: - stage: RegressionTest displayName: F# Compiler Regression Tests jobs: + # Note: This pipeline should be integrated with azure-pipelines-PR.yml EndToEndBuildTests job + # For now, keeping a minimal build job until architectural integration is completed - job: BuildCompiler - displayName: Build F# Compiler + displayName: Build F# Compiler (temporary - should use EndToEndBuildTests) pool: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals $(WindowsMachineQueueName) @@ -159,35 +161,16 @@ stages: exit 1 } - # Detect .NET SDK version from global.json if it exists - - task: PowerShell@2 - displayName: 'Detect .NET SDK version for $(DisplayName)' - inputs: - script: | - Set-Location $(Pipeline.Workspace)/TestRepo - - if (Test-Path "global.json") { - Write-Host "Found global.json, reading .NET SDK version" - $globalJson = Get-Content "global.json" | ConvertFrom-Json - $sdkVersion = $globalJson.sdk.version - Write-Host "Detected .NET SDK version: $sdkVersion" - Write-Host "##vso[task.setvariable variable=DotNetSdkVersion]$sdkVersion" - } else { - Write-Host "No global.json found, will use latest public SDK" - Write-Host "##vso[task.setvariable variable=DotNetSdkVersion]10.x" - } - - # Install appropriate .NET SDK version + # Install appropriate .NET SDK version using global.json if present - task: UseDotNet@2 displayName: 'Install .NET SDK for $(DisplayName)' inputs: packageType: sdk - version: $(DotNetSdkVersion) - includePreviewVersions: true + useGlobalJson: true workingDirectory: $(Pipeline.Workspace)/TestRepo installationPath: $(Pipeline.Workspace)/TestRepo/.dotnet - # Setup Directory.Build.props and UseLocalCompiler configuration using F# script + # Setup Directory.Build.props to import UseLocalCompiler configuration - task: PowerShell@2 displayName: 'Setup local compiler configuration for $(DisplayName)' inputs: @@ -200,7 +183,7 @@ stages: open System.IO open System.Xml - let useLocalCompilerImport = """""" + let useLocalCompilerImport = """""" let directoryBuildPropsPath = "Directory.Build.props" @@ -213,10 +196,10 @@ stages: let projectElement = doc.SelectSingleNode("/Project") if projectElement <> null then // Check if our import already exists - let existingImport = doc.SelectSingleNode(sprintf "//Import[@Project='UseLocalCompiler.Directory.Build.props']") + let existingImport = doc.SelectSingleNode(sprintf "//Import[@Project='$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props']") if existingImport = null then let importElement = doc.CreateElement("Import") - importElement.SetAttribute("Project", "UseLocalCompiler.Directory.Build.props") + importElement.SetAttribute("Project", "$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props") projectElement.InsertBefore(importElement, projectElement.FirstChild) |> ignore doc.Save(directoryBuildPropsPath) printfn "Added UseLocalCompiler import to existing Directory.Build.props" @@ -238,11 +221,7 @@ stages: # Run the F# script using dotnet fsi dotnet fsi PrepareRepoForTesting.fsx - # Copy UseLocalCompiler.Directory.Build.props as-is (path will be set via environment variable) - Copy-Item "$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props" "UseLocalCompiler.Directory.Build.props" -Force - - Write-Host "UseLocalCompiler.Directory.Build.props content (first 20 lines):" - Get-Content "UseLocalCompiler.Directory.Build.props" | Select-Object -First 20 + Write-Host "UseLocalCompiler.Directory.Build.props will be referenced from: $(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props" # Report dotnet info in test environment - task: PowerShell@2 @@ -310,14 +289,14 @@ stages: exit $exitCode } - # Capture and publish build logs on failure - # Publish only MSBuild binary logs + # Publish only MSBuild binary logs for efficient storage - task: PublishPipelineArtifact@1 displayName: 'Publish $(DisplayName) Binary Logs' inputs: targetPath: '$(Pipeline.Workspace)/TestRepo' artifactName: '$(DisplayName)_BinaryLogs' publishLocation: pipeline + artifact: '*.binlog' condition: always() continueOnError: true From c1b47750c06ab75e6f019c9f8c63e49aed145cd7 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 Aug 2025 10:37:02 +0000 Subject: [PATCH 6/9] Convert regression testing to Azure DevOps template and integrate with PR pipeline Co-authored-by: T-Gro <46543583+T-Gro@users.noreply.github.com> --- azure-pipelines-PR.yml | 26 +++ azure-pipelines-regression-test.yml | 258 ++----------------------- eng/templates/regression-test-jobs.yml | 236 ++++++++++++++++++++++ 3 files changed, 280 insertions(+), 240 deletions(-) create mode 100644 eng/templates/regression-test-jobs.yml diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index 9f06d744935..b4fff4db12f 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -738,6 +738,32 @@ stages: env: FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) displayName: End to end build tests + + # Publish artifacts for regression testing + - task: PublishPipelineArtifact@1 + displayName: Publish F# Compiler Artifacts for Regression Tests + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts' + artifactName: 'FSharpCompilerArtifacts' + publishLocation: pipeline + condition: succeeded() + + - task: PublishPipelineArtifact@1 + displayName: Publish UseLocalCompiler props file for Regression Tests + inputs: + targetPath: '$(Build.SourcesDirectory)/UseLocalCompiler.Directory.Build.props' + artifactName: 'UseLocalCompilerProps' + publishLocation: pipeline + condition: succeeded() + + # F# Compiler Regression Tests using third-party libraries + - template: /eng/templates/regression-test-jobs.yml + parameters: + testMatrix: + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.cmd + displayName: FSharpPlus # Up-to-date - disabled due to it being flaky #- job: UpToDate_Windows diff --git a/azure-pipelines-regression-test.yml b/azure-pipelines-regression-test.yml index f7c85634045..60adb339369 100644 --- a/azure-pipelines-regression-test.yml +++ b/azure-pipelines-regression-test.yml @@ -1,5 +1,10 @@ -# Regression Testing Pipeline -# Tests F# compiler regressions by building third-party F# projects with the freshly built compiler +# DEPRECATED: F# Compiler Regression Testing Pipeline +# This pipeline has been integrated into azure-pipelines-PR.yml +# The regression test logic is now available as a template at eng/templates/regression-test-jobs.yml +# and is automatically run as part of PR builds. +# +# This file is kept for documentation purposes and can be used as a standalone +# regression test pipeline if needed in the future. trigger: branches: @@ -41,25 +46,13 @@ variables: # Pick up pool provider name behavior from shared yaml template - template: /eng/common/templates/variables/pool-providers.yml -# Matrix of third-party repositories to test against -# This configuration can be extended in the future -parameters: -- name: testMatrix - type: object - default: - - repo: fsprojects/FSharpPlus - commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 - buildScript: build.cmd - displayName: FSharpPlus - stages: - stage: RegressionTest - displayName: F# Compiler Regression Tests + displayName: F# Compiler Regression Tests (Standalone) jobs: - # Note: This pipeline should be integrated with azure-pipelines-PR.yml EndToEndBuildTests job - # For now, keeping a minimal build job until architectural integration is completed + # Build the compiler to generate artifacts needed for regression testing - job: BuildCompiler - displayName: Build F# Compiler (temporary - should use EndToEndBuildTests) + displayName: Build F# Compiler for Regression Tests pool: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals $(WindowsMachineQueueName) @@ -102,226 +95,11 @@ stages: publishLocation: pipeline condition: succeeded() - # Test against third-party repositories - - ${{ each item in parameters.testMatrix }}: - - job: Test_${{ replace(item.repo, '/', '_') }} - displayName: 'Regression Test: ${{ item.displayName }}' - dependsOn: BuildCompiler - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - timeoutInMinutes: 60 - variables: - TestRepoName: ${{ item.repo }} - TestCommit: ${{ item.commit }} - BuildScript: ${{ item.buildScript }} - DisplayName: ${{ item.displayName }} - steps: - - checkout: none - displayName: Skip default checkout - - # Download the F# compiler artifacts - - task: DownloadPipelineArtifact@2 - displayName: Download F# Compiler Artifacts - inputs: - artifactName: 'FSharpCompilerArtifacts' - downloadPath: '$(Pipeline.Workspace)/FSharpCompiler' - - - task: DownloadPipelineArtifact@2 - displayName: Download UseLocalCompiler props - inputs: - artifactName: 'UseLocalCompilerProps' - downloadPath: '$(Pipeline.Workspace)/Props' - - # Checkout the third-party repository at specific commit - - task: PowerShell@2 - displayName: 'Checkout $(DisplayName) at specific commit' - inputs: - script: | - Write-Host "Cloning repository: $(TestRepoName)" - git clone https://github.com/$(TestRepoName).git $(Pipeline.Workspace)/TestRepo - Set-Location $(Pipeline.Workspace)/TestRepo - - Write-Host "Checking out commit: $(TestCommit)" - git checkout $(TestCommit) - - Write-Host "Successfully checked out $(TestRepoName) at commit $(TestCommit)" - git log -1 --oneline - - Write-Host "Repository structure:" - Get-ChildItem -Name - - Write-Host "Verifying build script exists: $(BuildScript)" - if (Test-Path "$(BuildScript)") { - Write-Host "✓ Build script found: $(BuildScript)" - } else { - Write-Host "✗ Build script not found: $(BuildScript)" - Write-Host "Available files in root:" - Get-ChildItem - exit 1 - } - - # Install appropriate .NET SDK version using global.json if present - - task: UseDotNet@2 - displayName: 'Install .NET SDK for $(DisplayName)' - inputs: - packageType: sdk - useGlobalJson: true - workingDirectory: $(Pipeline.Workspace)/TestRepo - installationPath: $(Pipeline.Workspace)/TestRepo/.dotnet - - # Setup Directory.Build.props to import UseLocalCompiler configuration - - task: PowerShell@2 - displayName: 'Setup local compiler configuration for $(DisplayName)' - inputs: - script: | - Set-Location $(Pipeline.Workspace)/TestRepo - - # Create F# script to handle Directory.Build.props setup - $fsharpScript = @' - #r "nuget: System.Xml.ReaderWriter" - open System.IO - open System.Xml - - let useLocalCompilerImport = """""" - - let directoryBuildPropsPath = "Directory.Build.props" - - if File.Exists(directoryBuildPropsPath) then - printfn "Directory.Build.props exists, modifying it" - let doc = XmlDocument() - doc.Load(directoryBuildPropsPath) - - // Find the Project element - let projectElement = doc.SelectSingleNode("/Project") - if projectElement <> null then - // Check if our import already exists - let existingImport = doc.SelectSingleNode(sprintf "//Import[@Project='$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props']") - if existingImport = null then - let importElement = doc.CreateElement("Import") - importElement.SetAttribute("Project", "$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props") - projectElement.InsertBefore(importElement, projectElement.FirstChild) |> ignore - doc.Save(directoryBuildPropsPath) - printfn "Added UseLocalCompiler import to existing Directory.Build.props" - else - printfn "UseLocalCompiler import already exists" - else - printfn "Warning: Could not find Project element in Directory.Build.props" - else - printfn "Creating new Directory.Build.props" - let content = sprintf "\n %s\n" useLocalCompilerImport - File.WriteAllText(directoryBuildPropsPath, content) - - printfn "Directory.Build.props content:" - File.ReadAllText(directoryBuildPropsPath) |> printfn "%s" - '@ - - $fsharpScript | Out-File -FilePath "PrepareRepoForTesting.fsx" -Encoding UTF8 - - # Run the F# script using dotnet fsi - dotnet fsi PrepareRepoForTesting.fsx - - Write-Host "UseLocalCompiler.Directory.Build.props will be referenced from: $(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props" - - # Report dotnet info in test environment - - task: PowerShell@2 - displayName: 'Report build environment for $(DisplayName)' - inputs: - script: | - Set-Location $(Pipeline.Workspace)/TestRepo - Write-Host "===========================================" - Write-Host "Environment Information for $(DisplayName)" - Write-Host "===========================================" - dotnet --info - Write-Host "" - Write-Host "MSBuild version:" - dotnet msbuild -version - Write-Host "" - Write-Host "F# Compiler artifacts available:" - Get-ChildItem "$(Pipeline.Workspace)\FSharpCompiler\bin\fsc\Release\net9.0" -Name - Write-Host "" - Write-Host "F# Core available:" - if (Test-Path "$(Pipeline.Workspace)\FSharpCompiler\bin\FSharp.Core\Release\netstandard2.0\FSharp.Core.dll") { - Write-Host "✓ FSharp.Core.dll found" - } else { - Write-Host "✗ FSharp.Core.dll not found" - } - Write-Host "" - Write-Host "Directory.Build.props content:" - Get-Content "Directory.Build.props" - Write-Host "" - Write-Host "===========================================" - - # Build the third-party project using local F# compiler - - task: PowerShell@2 - displayName: 'Build $(DisplayName) with local F# compiler' - env: - # Set environment variables to use local compiler and enforce binary logs - LocalFSharpCompilerPath: $(Pipeline.Workspace)/FSharpCompiler - LoadLocalFSharpBuild: true - LocalFSharpCompilerConfiguration: Release - # Force MSBuild binary logs - MSBUILDBINARYLOGGERENABLED: true - MSBUILDBINARYLOGGER: "*.binlog" - timeoutInMinutes: 45 - inputs: - script: | - Set-Location $(Pipeline.Workspace)/TestRepo - Write-Host "============================================" - Write-Host "Starting build for $(DisplayName)" - Write-Host "Repository: $(TestRepoName)" - Write-Host "Commit: $(TestCommit)" - Write-Host "Build Script: $(BuildScript)" - Write-Host "============================================" - Write-Host "" - - Write-Host "Executing: $(BuildScript)" - cmd /c "$(BuildScript)" - $exitCode = $LASTEXITCODE - - Write-Host "" - Write-Host "============================================" - Write-Host "Build completed for $(DisplayName)" - Write-Host "Exit code: $exitCode" - Write-Host "============================================" - - if ($exitCode -ne 0) { - exit $exitCode - } - - # Publish only MSBuild binary logs for efficient storage - - task: PublishPipelineArtifact@1 - displayName: 'Publish $(DisplayName) Binary Logs' - inputs: - targetPath: '$(Pipeline.Workspace)/TestRepo' - artifactName: '$(DisplayName)_BinaryLogs' - publishLocation: pipeline - artifact: '*.binlog' - condition: always() - continueOnError: true - - # Report success/failure - - task: PowerShell@2 - displayName: 'Report $(DisplayName) test result' - condition: always() - inputs: - script: | - Set-Location $(Pipeline.Workspace)/TestRepo - Write-Host "" - Write-Host "============================================" - Write-Host "Regression test completed for $(DisplayName)" - Write-Host "Repository: $(TestRepoName)" - Write-Host "Commit: $(TestCommit)" - Write-Host "Build Script: $(BuildScript)" - if ($env:AGENT_JOBSTATUS -eq "Succeeded") { - Write-Host "Status: ✓ SUCCESS" - Write-Host "The $(DisplayName) library builds successfully with the new F# compiler" - } else { - Write-Host "Status: ✗ FAILED" - Write-Host "The $(DisplayName) library failed to build with the new F# compiler" - Write-Host "Check the build logs and artifacts for details" - } - Write-Host "============================================" - - Write-Host "Binary logs found:" - Get-ChildItem "*.binlog" -ErrorAction SilentlyContinue | ForEach-Object { Write-Host $_.Name } \ No newline at end of file + # Use the regression test template + - template: /eng/templates/regression-test-jobs.yml + parameters: + testMatrix: + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.cmd + displayName: FSharpPlus \ No newline at end of file diff --git a/eng/templates/regression-test-jobs.yml b/eng/templates/regression-test-jobs.yml new file mode 100644 index 00000000000..ca67a83794d --- /dev/null +++ b/eng/templates/regression-test-jobs.yml @@ -0,0 +1,236 @@ +# Template for F# Compiler Regression Tests +# Tests third-party F# projects with the freshly built compiler + +parameters: +- name: testMatrix + type: object + default: + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.cmd + displayName: FSharpPlus + +jobs: +# Test against third-party repositories +- ${{ each item in parameters.testMatrix }}: + - job: RegressionTest_${{ replace(item.repo, '/', '_') }} + displayName: 'Regression Test: ${{ item.displayName }}' + dependsOn: EndToEndBuildTests + pool: + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 60 + variables: + TestRepoName: ${{ item.repo }} + TestCommit: ${{ item.commit }} + BuildScript: ${{ item.buildScript }} + DisplayName: ${{ item.displayName }} + steps: + - checkout: none + displayName: Skip default checkout + + # Download the F# compiler artifacts from EndToEndBuildTests job + - task: DownloadPipelineArtifact@2 + displayName: Download F# Compiler Artifacts + inputs: + artifactName: 'FSharpCompilerArtifacts' + downloadPath: '$(Pipeline.Workspace)/FSharpCompiler' + + - task: DownloadPipelineArtifact@2 + displayName: Download UseLocalCompiler props + inputs: + artifactName: 'UseLocalCompilerProps' + downloadPath: '$(Pipeline.Workspace)/Props' + + # Checkout the third-party repository at specific commit + - task: PowerShell@2 + displayName: 'Checkout $(DisplayName) at specific commit' + inputs: + script: | + Write-Host "Cloning repository: $(TestRepoName)" + git clone https://github.com/$(TestRepoName).git $(Pipeline.Workspace)/TestRepo + Set-Location $(Pipeline.Workspace)/TestRepo + + Write-Host "Checking out commit: $(TestCommit)" + git checkout $(TestCommit) + + Write-Host "Successfully checked out $(TestRepoName) at commit $(TestCommit)" + git log -1 --oneline + + Write-Host "Repository structure:" + Get-ChildItem -Name + + Write-Host "Verifying build script exists: $(BuildScript)" + if (Test-Path "$(BuildScript)") { + Write-Host "✓ Build script found: $(BuildScript)" + } else { + Write-Host "✗ Build script not found: $(BuildScript)" + Write-Host "Available files in root:" + Get-ChildItem + exit 1 + } + + # Install appropriate .NET SDK version using global.json if present + - task: UseDotNet@2 + displayName: 'Install .NET SDK for $(DisplayName)' + inputs: + packageType: sdk + useGlobalJson: true + workingDirectory: $(Pipeline.Workspace)/TestRepo + installationPath: $(Pipeline.Workspace)/TestRepo/.dotnet + + # Setup Directory.Build.props to import UseLocalCompiler configuration + - task: PowerShell@2 + displayName: 'Setup local compiler configuration for $(DisplayName)' + inputs: + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + + # Create F# script to handle Directory.Build.props setup + $fsharpScript = @' + #r "nuget: System.Xml.ReaderWriter" + open System.IO + open System.Xml + + let useLocalCompilerImport = """""" + + let directoryBuildPropsPath = "Directory.Build.props" + + if File.Exists(directoryBuildPropsPath) then + printfn "Directory.Build.props exists, modifying it" + let doc = XmlDocument() + doc.Load(directoryBuildPropsPath) + + // Find the Project element + let projectElement = doc.SelectSingleNode("/Project") + if projectElement <> null then + // Check if our import already exists + let existingImport = doc.SelectSingleNode(sprintf "//Import[@Project='$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props']") + if existingImport = null then + let importElement = doc.CreateElement("Import") + importElement.SetAttribute("Project", "$(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props") + projectElement.InsertBefore(importElement, projectElement.FirstChild) |> ignore + doc.Save(directoryBuildPropsPath) + printfn "Added UseLocalCompiler import to existing Directory.Build.props" + else + printfn "UseLocalCompiler import already exists" + else + printfn "Warning: Could not find Project element in Directory.Build.props" + else + printfn "Creating new Directory.Build.props" + let content = sprintf "\n %s\n" useLocalCompilerImport + File.WriteAllText(directoryBuildPropsPath, content) + + printfn "Directory.Build.props content:" + File.ReadAllText(directoryBuildPropsPath) |> printfn "%s" + '@ + + $fsharpScript | Out-File -FilePath "PrepareRepoForTesting.fsx" -Encoding UTF8 + + # Run the F# script using dotnet fsi + dotnet fsi PrepareRepoForTesting.fsx + + Write-Host "UseLocalCompiler.Directory.Build.props will be referenced from: $(Pipeline.Workspace)/Props/UseLocalCompiler.Directory.Build.props" + + # Report dotnet info in test environment + - task: PowerShell@2 + displayName: 'Report build environment for $(DisplayName)' + inputs: + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "===========================================" + Write-Host "Environment Information for $(DisplayName)" + Write-Host "===========================================" + dotnet --info + Write-Host "" + Write-Host "MSBuild version:" + dotnet msbuild -version + Write-Host "" + Write-Host "F# Compiler artifacts available:" + Get-ChildItem "$(Pipeline.Workspace)\FSharpCompiler\bin\fsc\Release\net9.0" -Name + Write-Host "" + Write-Host "F# Core available:" + if (Test-Path "$(Pipeline.Workspace)\FSharpCompiler\bin\FSharp.Core\Release\netstandard2.0\FSharp.Core.dll") { + Write-Host "✓ FSharp.Core.dll found" + } else { + Write-Host "✗ FSharp.Core.dll not found" + } + Write-Host "" + Write-Host "Directory.Build.props content:" + Get-Content "Directory.Build.props" + Write-Host "" + Write-Host "===========================================" + + # Build the third-party project using local F# compiler + - task: PowerShell@2 + displayName: 'Build $(DisplayName) with local F# compiler' + env: + # Set environment variables to use local compiler and enforce binary logs + LocalFSharpCompilerPath: $(Pipeline.Workspace)/FSharpCompiler + LoadLocalFSharpBuild: true + LocalFSharpCompilerConfiguration: Release + # Force MSBuild binary logs + MSBUILDBINARYLOGGERENABLED: true + MSBUILDBINARYLOGGER: "*.binlog" + timeoutInMinutes: 45 + inputs: + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "============================================" + Write-Host "Starting build for $(DisplayName)" + Write-Host "Repository: $(TestRepoName)" + Write-Host "Commit: $(TestCommit)" + Write-Host "Build Script: $(BuildScript)" + Write-Host "============================================" + Write-Host "" + + Write-Host "Executing: $(BuildScript)" + cmd /c "$(BuildScript)" + $exitCode = $LASTEXITCODE + + Write-Host "" + Write-Host "============================================" + Write-Host "Build completed for $(DisplayName)" + Write-Host "Exit code: $exitCode" + Write-Host "============================================" + + if ($exitCode -ne 0) { + exit $exitCode + } + + # Publish only MSBuild binary logs for efficient storage + - task: PublishPipelineArtifact@1 + displayName: 'Publish $(DisplayName) Binary Logs' + inputs: + targetPath: '$(Pipeline.Workspace)/TestRepo' + artifactName: '$(DisplayName)_BinaryLogs' + publishLocation: pipeline + artifact: '*.binlog' + condition: always() + continueOnError: true + + # Report success/failure + - task: PowerShell@2 + displayName: 'Report $(DisplayName) test result' + condition: always() + inputs: + script: | + Set-Location $(Pipeline.Workspace)/TestRepo + Write-Host "" + Write-Host "============================================" + Write-Host "Regression test completed for $(DisplayName)" + Write-Host "Repository: $(TestRepoName)" + Write-Host "Commit: $(TestCommit)" + Write-Host "Build Script: $(BuildScript)" + if ($env:AGENT_JOBSTATUS -eq "Succeeded") { + Write-Host "Status: ✓ SUCCESS" + Write-Host "The $(DisplayName) library builds successfully with the new F# compiler" + } else { + Write-Host "Status: ✗ FAILED" + Write-Host "The $(DisplayName) library failed to build with the new F# compiler" + Write-Host "Check the build logs and artifacts for details" + } + Write-Host "============================================" + + Write-Host "Binary logs found:" + Get-ChildItem "*.binlog" -ErrorAction SilentlyContinue | ForEach-Object { Write-Host $_.Name } \ No newline at end of file From 5ca6ca8bf04e8edc5caded173150bbc82832d39b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:43:00 +0000 Subject: [PATCH 7/9] Address Azure DevOps template feedback - remove strategy matrix, fix job naming, clean up parameters Co-authored-by: T-Gro <46543583+T-Gro@users.noreply.github.com> --- azure-pipelines-PR.yml | 10 --- azure-pipelines-regression-test.yml | 105 ------------------------- docs/regression-testing-pipeline.md | 96 +++++++++++----------- eng/templates/regression-test-jobs.yml | 7 +- 4 files changed, 53 insertions(+), 165 deletions(-) delete mode 100644 azure-pipelines-regression-test.yml diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index b4fff4db12f..ab70e850d97 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -720,23 +720,13 @@ stages: pool: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals $(WindowsMachineQueueName) - strategy: - maxParallel: 2 - matrix: - regular: - _experimental_flag: '' - experimental_features: - _experimental_flag: '' steps: - checkout: self clean: true - script: .\Build.cmd -c Release -pack env: NativeToolsOnMachine: true - FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c Release - env: - FSHARP_EXPERIMENTAL_FEATURES: $(_experimental_flag) displayName: End to end build tests # Publish artifacts for regression testing diff --git a/azure-pipelines-regression-test.yml b/azure-pipelines-regression-test.yml deleted file mode 100644 index 60adb339369..00000000000 --- a/azure-pipelines-regression-test.yml +++ /dev/null @@ -1,105 +0,0 @@ -# DEPRECATED: F# Compiler Regression Testing Pipeline -# This pipeline has been integrated into azure-pipelines-PR.yml -# The regression test logic is now available as a template at eng/templates/regression-test-jobs.yml -# and is automatically run as part of PR builds. -# -# This file is kept for documentation purposes and can be used as a standalone -# regression test pipeline if needed in the future. - -trigger: - branches: - include: - - main - - release/* - - feature/* - paths: - include: - - src/* - exclude: - - docs/* - - .github/* - - '*.md' - -pr: - branches: - include: - - main - - release/* - - feature/* - paths: - include: - - src/* - exclude: - - docs/* - - .github/* - - '*.md' - -variables: - - name: _TeamName - value: FSharp - - name: _BuildConfig - value: Release - - name: _PublishUsingPipelines - value: true - - name: Codeql.Enabled - value: false # Disabled for regression tests - # Pick up pool provider name behavior from shared yaml template - - template: /eng/common/templates/variables/pool-providers.yml - -stages: -- stage: RegressionTest - displayName: F# Compiler Regression Tests (Standalone) - jobs: - # Build the compiler to generate artifacts needed for regression testing - - job: BuildCompiler - displayName: Build F# Compiler for Regression Tests - pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals $(WindowsMachineQueueName) - timeoutInMinutes: 60 - steps: - - checkout: self - clean: true - displayName: Checkout F# compiler source - - - task: UseDotNet@2 - displayName: install SDK - inputs: - packageType: sdk - version: '10.x' - includePreviewVersions: true - workingDirectory: $(Build.SourcesDirectory) - installationPath: $(Build.SourcesDirectory)/.dotnet - - - script: .\eng\common\dotnet.cmd - displayName: Ensure correct .NET SDK version - - - script: .\Build.cmd -c $(_BuildConfig) -pack - env: - NativeToolsOnMachine: true - displayName: Build F# compiler and create packages - - - task: PublishPipelineArtifact@1 - displayName: Publish F# Compiler Artifacts - inputs: - targetPath: '$(Build.SourcesDirectory)/artifacts' - artifactName: 'FSharpCompilerArtifacts' - publishLocation: pipeline - condition: succeeded() - - - task: PublishPipelineArtifact@1 - displayName: Publish UseLocalCompiler props file - inputs: - targetPath: '$(Build.SourcesDirectory)/UseLocalCompiler.Directory.Build.props' - artifactName: 'UseLocalCompilerProps' - publishLocation: pipeline - condition: succeeded() - - # Use the regression test template - - template: /eng/templates/regression-test-jobs.yml - parameters: - testMatrix: - - repo: fsprojects/FSharpPlus - commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 - buildScript: build.cmd - displayName: FSharpPlus \ No newline at end of file diff --git a/docs/regression-testing-pipeline.md b/docs/regression-testing-pipeline.md index d0fe722d91d..7d617fb4469 100644 --- a/docs/regression-testing-pipeline.md +++ b/docs/regression-testing-pipeline.md @@ -1,24 +1,35 @@ -# F# Compiler Regression Testing Pipeline +# F# Compiler Regression Testing -This document describes the F# compiler regression testing pipeline implemented in `azure-pipelines-regression-test.yml`. +This document describes the F# compiler regression testing functionality implemented as a reusable Azure DevOps template in `eng/templates/regression-test-jobs.yml` and integrated into the main PR pipeline (`azure-pipelines-PR.yml`). ## Purpose -The regression testing pipeline helps catch F# compiler regressions by building popular third-party F# libraries with the freshly built compiler from this repository. This provides early detection of breaking changes that might affect real-world F# projects. +The regression testing helps catch F# compiler regressions by building popular third-party F# libraries with the freshly built compiler from this repository. This provides early detection of breaking changes that might affect real-world F# projects. ## How It Works -### Pipeline Workflow +### Integration with PR Pipeline -1. **Build F# Compiler**: The pipeline first builds the F# compiler and packages from the current source code using the standard build process. +The regression tests are automatically run as part of every PR build, depending on the `EndToEndBuildTests` job for the F# compiler artifacts. -2. **Test Against Third-Party Libraries**: For each library in the test matrix: - - Checkout the third-party repository at a specific commit - - Inject the `UseLocalCompiler.Directory.Build.props` configuration to use the locally built compiler - - Run the library's build script - - Capture detailed build logs and artifacts +### Template-Based Architecture + +The regression testing logic is implemented as a reusable Azure DevOps template that can be consumed by multiple pipelines: + +- **Template Location**: `eng/templates/regression-test-jobs.yml` +- **Integration**: Called from `azure-pipelines-PR.yml` +- **Dependencies**: Depends on `EndToEndBuildTests` job for compiler artifacts + +### Workflow -3. **Report Results**: Success/failure status is reported with detailed logs for diagnosis. +1. **Build F# Compiler**: The `EndToEndBuildTests` job builds the F# compiler and publishes required artifacts +2. **Matrix Execution**: For each library in the test matrix (running in parallel): + - Checkout the third-party repository at a specific commit + - Install appropriate .NET SDK version using the repository's `global.json` + - Setup `Directory.Build.props` to import `UseLocalCompiler.Directory.Build.props` + - Build the library using its standard build script + - Publish MSBuild binary logs for analysis +3. **Report Results**: Success/failure status is reported with build logs for diagnosis ### Key Features @@ -37,54 +48,51 @@ The pipeline currently tests against: ## Adding New Libraries -To add a new library to the test matrix: - -1. **Choose a Library**: Select a representative F# library that uses features you want to test. - -2. **Find a Stable Commit**: Choose a specific commit SHA that is known to build successfully with the current F# compiler. - -3. **Update the Matrix**: Add an entry to the `testMatrix` parameter in the pipeline: +To add a new library to the test matrix, update the template invocation in `azure-pipelines-PR.yml`: ```yaml -parameters: -- name: testMatrix - type: object - default: - - repo: fsprojects/FSharpPlus - commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 - buildScript: build.cmd - displayName: FSharpPlus - - repo: your-org/your-library # Add your library here - commit: abc123def456... # Specific commit SHA - buildScript: build.sh # Build script (build.cmd, build.sh, etc.) - displayName: YourLibrary # Human-readable name +# F# Compiler Regression Tests using third-party libraries +- template: /eng/templates/regression-test-jobs.yml + parameters: + testMatrix: + - repo: fsprojects/FSharpPlus + commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 + buildScript: build.cmd + displayName: FSharpPlus + - repo: your-org/your-library # Add your library here + commit: abc123def456... # Specific commit SHA + buildScript: build.sh # Build script (build.cmd, build.sh, etc.) + displayName: YourLibrary # Human-readable name ``` -4. **Test the Configuration**: Verify that your library builds correctly with the current compiler before adding it to the matrix. +Each test matrix entry requires: +- **repo**: GitHub repository in `owner/name` format +- **commit**: Specific commit SHA for reproducible results +- **buildScript**: Build script to execute (e.g., `build.cmd`, `build.sh`) +- **displayName**: Human-readable name for the job ## Pipeline Configuration ### Triggers -The pipeline is triggered by: -- **Branches**: main, release/*, feature/* -- **Paths**: Changes to compiler source code (src/Compiler/, src/fsc/, src/FSharp.Core/, src/FSharp.Build/) -- **Exclusions**: Documentation and non-compiler changes +Regression tests run automatically as part of PR builds when: +- **PR Pipeline**: Triggered by pull requests to main branches +- **Dependencies**: Runs after `EndToEndBuildTests` completes successfully +- **Parallel Execution**: Each repository in the test matrix runs as a separate job in parallel ### Build Environment -- **OS**: Windows (windows.vs2022.amd64.open) -- **Pool**: Uses the standard public build pool (`DncEngPublicBuildPool`) -- **Timeout**: 60 minutes per job -- **.NET SDK**: Automatically installs the required preview SDK +- **OS**: Windows (using `$(WindowsMachineQueueName)`) +- **Pool**: Standard public build pool (`$(DncEngPublicBuildPool)`) +- **Timeout**: 60 minutes per regression test job +- **.NET SDK**: Automatically detects and installs SDK version from each repository's `global.json` ### Artifacts -The pipeline publishes several artifacts for analysis: -- **FSharpCompilerArtifacts**: Complete F# compiler build output -- **UseLocalCompilerProps**: Configuration file for using local compiler -- **{Library}_BuildOutput**: Complete build output from each tested library -- **{Library}_BinaryLogs**: MSBuild binary logs when available +The regression tests publish focused artifacts for analysis: +- **FSharpCompilerArtifacts**: F# compiler build output (from `EndToEndBuildTests`) +- **UseLocalCompilerProps**: Configuration file for using local compiler (from `EndToEndBuildTests`) +- **{LibraryName}_BinaryLogs**: MSBuild binary logs from each tested library for efficient diagnosis ## Troubleshooting Build Failures diff --git a/eng/templates/regression-test-jobs.yml b/eng/templates/regression-test-jobs.yml index ca67a83794d..e859e8a190c 100644 --- a/eng/templates/regression-test-jobs.yml +++ b/eng/templates/regression-test-jobs.yml @@ -4,17 +4,12 @@ parameters: - name: testMatrix type: object - default: - - repo: fsprojects/FSharpPlus - commit: f614035b75922aba41ed6a36c2fc986a2171d2b8 - buildScript: build.cmd - displayName: FSharpPlus jobs: # Test against third-party repositories - ${{ each item in parameters.testMatrix }}: - job: RegressionTest_${{ replace(item.repo, '/', '_') }} - displayName: 'Regression Test: ${{ item.displayName }}' + displayName: 'Regression Test: ${{ item.repo }}' dependsOn: EndToEndBuildTests pool: name: $(DncEngPublicBuildPool) From 05688c7334dd1802916e1af5a20ef90af6b25251 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 Aug 2025 13:58:13 +0000 Subject: [PATCH 8/9] Reduce artifact size by publishing only required compiler directories Co-authored-by: T-Gro <46543583+T-Gro@users.noreply.github.com> --- azure-pipelines-PR.yml | 14 +++++++++++--- eng/templates/regression-test-jobs.yml | 12 +++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index ab70e850d97..e0a5db89e6c 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -731,10 +731,18 @@ stages: # Publish artifacts for regression testing - task: PublishPipelineArtifact@1 - displayName: Publish F# Compiler Artifacts for Regression Tests + displayName: Publish F# Compiler FSC Artifacts for Regression Tests inputs: - targetPath: '$(Build.SourcesDirectory)/artifacts' - artifactName: 'FSharpCompilerArtifacts' + targetPath: '$(Build.SourcesDirectory)/artifacts/bin/fsc' + artifactName: 'FSharpCompilerFscArtifacts' + publishLocation: pipeline + condition: succeeded() + + - task: PublishPipelineArtifact@1 + displayName: Publish F# Core Artifacts for Regression Tests + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts/bin/FSharp.Core' + artifactName: 'FSharpCoreArtifacts' publishLocation: pipeline condition: succeeded() diff --git a/eng/templates/regression-test-jobs.yml b/eng/templates/regression-test-jobs.yml index e859e8a190c..2f7cf20ec4c 100644 --- a/eng/templates/regression-test-jobs.yml +++ b/eng/templates/regression-test-jobs.yml @@ -26,10 +26,16 @@ jobs: # Download the F# compiler artifacts from EndToEndBuildTests job - task: DownloadPipelineArtifact@2 - displayName: Download F# Compiler Artifacts + displayName: Download F# Compiler FSC Artifacts inputs: - artifactName: 'FSharpCompilerArtifacts' - downloadPath: '$(Pipeline.Workspace)/FSharpCompiler' + artifactName: 'FSharpCompilerFscArtifacts' + downloadPath: '$(Pipeline.Workspace)/FSharpCompiler/bin/fsc' + + - task: DownloadPipelineArtifact@2 + displayName: Download F# Core Artifacts + inputs: + artifactName: 'FSharpCoreArtifacts' + downloadPath: '$(Pipeline.Workspace)/FSharpCompiler/bin/FSharp.Core' - task: DownloadPipelineArtifact@2 displayName: Download UseLocalCompiler props From 48832d21e68b2445b38b9d4451856e4788a50563 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:07:39 +0000 Subject: [PATCH 9/9] Fix Azure DevOps template invocation - move to correct jobs level Co-authored-by: T-Gro <46543583+T-Gro@users.noreply.github.com> --- azure-pipelines-PR.yml | 2 +- eng/templates/regression-test-jobs.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index e0a5db89e6c..014e5fb1d40 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -754,7 +754,7 @@ stages: publishLocation: pipeline condition: succeeded() - # F# Compiler Regression Tests using third-party libraries + # F# Compiler Regression Tests using third-party libraries - template: /eng/templates/regression-test-jobs.yml parameters: testMatrix: diff --git a/eng/templates/regression-test-jobs.yml b/eng/templates/regression-test-jobs.yml index 2f7cf20ec4c..d99841d01f1 100644 --- a/eng/templates/regression-test-jobs.yml +++ b/eng/templates/regression-test-jobs.yml @@ -206,7 +206,6 @@ jobs: targetPath: '$(Pipeline.Workspace)/TestRepo' artifactName: '$(DisplayName)_BinaryLogs' publishLocation: pipeline - artifact: '*.binlog' condition: always() continueOnError: true