From 2ff1b606297295698e48fc51d2b39b4f3ac5815d Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 16 Jan 2026 10:06:58 +0100 Subject: [PATCH 1/2] [devops] Fix a few issues with pwsh tests and test result parsing. * Fix an issue parsing test results, causing errors like: Failed to compute test summaries: Method invocation failed because [System.Char] does not contain a method named 'Contains'. * Fix tests related to checking if a commit is the last in a PR. --- .../automation/scripts/GitHub.Tests.ps1 | 26 ++++++++--------- tools/devops/automation/scripts/GitHub.psm1 | 28 +++++++++++++++++++ .../automation/scripts/TestResults.Tests.ps1 | 5 ++-- .../automation/scripts/TestResults.psm1 | 22 ++++++++++++--- 4 files changed, 59 insertions(+), 22 deletions(-) diff --git a/tools/devops/automation/scripts/GitHub.Tests.ps1 b/tools/devops/automation/scripts/GitHub.Tests.ps1 index abfc73d2c23f..ea6b26f8c2cf 100644 --- a/tools/devops/automation/scripts/GitHub.Tests.ps1 +++ b/tools/devops/automation/scripts/GitHub.Tests.ps1 @@ -247,11 +247,7 @@ Describe 'IsCurrentCommitLatestInPR' { } } -ModuleName 'GitHub' - $githubComments = [GitHubComments]::new("testorg", "testrepo", "test-token", "abc123def456") - $githubComments.PRIds = @("123") - - $result = $githubComments.IsCurrentCommitLatestInPR() - + $result = Get-IsCurrentCommitLatestInPR -Org "testorg" -Repo "testrepo" -Token "test-token" -Hash "abc123def456" -PrIDs @("123") $result | Should -Be $true } @@ -264,20 +260,20 @@ Describe 'IsCurrentCommitLatestInPR' { } } -ModuleName 'GitHub' - $githubComments = [GitHubComments]::new("testorg", "testrepo", "test-token", "abc123def456") - $githubComments.PRIds = @("123") - - $result = $githubComments.IsCurrentCommitLatestInPR() - + $result = Get-IsCurrentCommitLatestInPR -Org "testorg" -Repo "testrepo" -Token "test-token" -Hash "abc123def456" -PrIDs @("123") $result | Should -Be $false } It 'returns true when not in PR context' { - $githubComments = [GitHubComments]::new("testorg", "testrepo", "test-token", "abc123def456") - $githubComments.PRIds = @() # Empty array means not in PR - - $result = $githubComments.IsCurrentCommitLatestInPR() - + Mock Invoke-Request { + return @{ + "head" = @{ + "sha" = "different123hash" + } + } + } -ModuleName 'GitHub' + + $result = Get-IsCurrentCommitLatestInPR -Org "testorg" -Repo "testrepo" -Token "test-token" -Hash "abc123def456" -PrIDs @() # Empty array means not in PR $result | Should -Be $true } } diff --git a/tools/devops/automation/scripts/GitHub.psm1 b/tools/devops/automation/scripts/GitHub.psm1 index adf8ad0c9b90..5f8ad2bc9190 100644 --- a/tools/devops/automation/scripts/GitHub.psm1 +++ b/tools/devops/automation/scripts/GitHub.psm1 @@ -1305,6 +1305,33 @@ function Convert-Markdown { return $InputContents } +function Get-IsCurrentCommitLatestInPR { + param ( + [ValidateNotNullOrEmpty ()] + [string] + $Org, + + [ValidateNotNullOrEmpty ()] + [string] + $Repo, + + [ValidateNotNullOrEmpty ()] + [string] + $Token, + + [string] + $Hash, + + [string[]] + $PrIDs + ) + + $githubComments = New-GitHubCommentsObject -Org $Org -Repo $Repo -Token $Token -Hash $Hash + $githubComments.PRIds = $PrIDs + $result = $githubComments.IsCurrentCommitLatestInPR() + return $result +} + # module exports, any other functions are private and should not be used outside the module. Export-ModuleMember -Function New-GitHubComment Export-ModuleMember -Function Get-GitHubPRInfo @@ -1313,6 +1340,7 @@ Export-ModuleMember -Function New-GistWithFiles Export-ModuleMember -Function New-GistObjectDefinition Export-ModuleMember -Function New-GistWithContent Export-ModuleMember -Function Convert-Markdown +Export-ModuleMember -Function Get-IsCurrentCommitLatestInPR # new future API that uses objects. Export-ModuleMember -Function New-GitHubCommentsObject diff --git a/tools/devops/automation/scripts/TestResults.Tests.ps1 b/tools/devops/automation/scripts/TestResults.Tests.ps1 index a0fc6bcfc604..ddbb54ee35f9 100644 --- a/tools/devops/automation/scripts/TestResults.Tests.ps1 +++ b/tools/devops/automation/scripts/TestResults.Tests.ps1 @@ -915,7 +915,7 @@ Describe "TestResults tests" { 1 tests failed, 0 tests passed.
- +# :tada: All 5 tests passed :tada:
[Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_maccatalyst-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_maccatalyst-1&api-version=6.0&`$format=zip) @@ -970,9 +970,8 @@ Describe "TestResults tests" { ### :x: dotnettests tests (MacCatalyst) -5 tests failed, 6 tests passed.
- +5 tests failed, 6 tests passed.
[Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_maccatalyst-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_maccatalyst-1&api-version=6.0&`$format=zip) diff --git a/tools/devops/automation/scripts/TestResults.psm1 b/tools/devops/automation/scripts/TestResults.psm1 index 9445337b08b5..48b19a1d0e3d 100644 --- a/tools/devops/automation/scripts/TestResults.psm1 +++ b/tools/devops/automation/scripts/TestResults.psm1 @@ -393,13 +393,22 @@ class ParallelTestsResults { $stringBuilder.AppendLine("") } else { $addSummary = $true - $startLine = 0 + $addDetails = $true + $startLine = -1 if (Test-Path -Path $r.ResultsPath -PathType Leaf) { - $resultLines = Get-Content -Path $r.ResultsPath + $resultLines = @(Get-Content -Path $r.ResultsPath) for ($i = 0; $i -lt $resultLines.Length; $i++) { $line = $resultLines[$i] if ($line.Contains("
")) { - $startLine = $i + if ($startLine -eq -1) { + $startLine = $i + } + $addDetails = $false + break + } elseif ($line.Contains("")) { + if ($startLine -eq -1) { + $startLine = $i + } $addSummary = $false break } elseif ($line.Contains("## Failed tests")) { @@ -413,12 +422,17 @@ class ParallelTestsResults { if ($addSummary) { $stringBuilder.AppendLine("$($result.Failed) tests failed, $($result.Passed) tests passed.") + } + if ($addDetails) { $stringBuilder.AppendLine("
") } + if ($startLine -eq -1) { + $startLine = 0 + } for ($i = $startLine; $i -lt $resultLines.Length; $i++) { $stringBuilder.AppendLine($resultLines[$i]) } - if ($addSummary) { + if ($addDetails) { $stringBuilder.AppendLine("
") } From f0dc8ba4cd9c4d1e2c7793ecfd130eeb31c58d7a Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Fri, 16 Jan 2026 18:00:03 +0100 Subject: [PATCH 2/2] More fixes --- .../automation/scripts/TestResults.Tests.ps1 | 116 +++++++++++++++++- .../automation/scripts/TestResults.psm1 | 5 +- 2 files changed, 115 insertions(+), 6 deletions(-) diff --git a/tools/devops/automation/scripts/TestResults.Tests.ps1 b/tools/devops/automation/scripts/TestResults.Tests.ps1 index ddbb54ee35f9..d5a6d30af00c 100644 --- a/tools/devops/automation/scripts/TestResults.Tests.ps1 +++ b/tools/devops/automation/scripts/TestResults.Tests.ps1 @@ -165,6 +165,95 @@ Describe "TestResults tests" { } } } +"@ + + $stageDependenciesWithMoreFailingTests = @" +{ + "build_macos_tests": { + "build_macos_tests_job": { + "result": "Succeeded" + } + }, + "configure_build": { + "configure": { + "outputs": { + "test_matrix.TEST_MATRIX": "$($matrix.Replace("`n", "\n").Replace("`"", "\`""))" + } + } + }, + "simulator_tests": { + "tests": { + "outputs": { + "dotnettests_tvos.runTests.TESTS_JOBSTATUS": "Succeeded", + "dotnettests_tvos.fix_commit.GIT_HASH": "fa3d1deb4e2d0ac358f2e0ac80e3d305ca541848", + "dotnettests_tvos.DownloadPipelineArtifact1.BuildNumber": "8894907", + "dotnettests_tvos.Bash23.TESTS_ATTEMPT": "1", + "dotnettests_tvos.Bash23.TESTS_BOT": "XAMMINI-012.Ventura", + "dotnettests_tvos.Bash23.TESTS_JOBSTATUS": "Failed", + "dotnettests_tvos.Bash23.TESTS_LABEL": "dotnettests", + "dotnettests_tvos.Bash23.TESTS_PLATFORM": "", + "dotnettests_tvos.Bash23.TESTS_TITLE": "dotnettests_tvos", + "dotnettests_tvos.DownloadPipelineArtifact2.BuildNumber": "8894907", + "dotnettests_maccatalyst.fix_commit.GIT_HASH": "fa3d1deb4e2d0ac358f2e0ac80e3d305ca541848", + "dotnettests_maccatalyst.DownloadPipelineArtifact1.BuildNumber": "8894907", + "dotnettests_maccatalyst.Bash23.TESTS_ATTEMPT": "1", + "dotnettests_maccatalyst.Bash23.TESTS_BOT": "XAMBOT-1023.Ventura", + "dotnettests_maccatalyst.Bash23.TESTS_JOBSTATUS": "Failed", + "dotnettests_maccatalyst.Bash23.TESTS_LABEL": "dotnettests", + "dotnettests_maccatalyst.Bash23.TESTS_PLATFORM": "", + "dotnettests_maccatalyst.Bash23.TESTS_TITLE": "dotnettests_maccatalyst", + "dotnettests_maccatalyst.DownloadPipelineArtifact2.BuildNumber": "8894907", + "dotnettests_macos.fix_commit.GIT_HASH": "fa3d1deb4e2d0ac358f2e0ac80e3d305ca541848", + "dotnettests_macos.DownloadPipelineArtifact1.BuildNumber": "8894907", + "dotnettests_macos.Bash23.TESTS_ATTEMPT": "1", + "dotnettests_macos.Bash23.TESTS_BOT": "XAMMINI-015.Ventura", + "dotnettests_macos.Bash23.TESTS_JOBSTATUS": "Failed", + "dotnettests_macos.Bash23.TESTS_LABEL": "dotnettests", + "dotnettests_macos.Bash23.TESTS_PLATFORM": "", + "dotnettests_macos.Bash23.TESTS_TITLE": "dotnettests_macos", + "dotnettests_macos.DownloadPipelineArtifact2.BuildNumber": "8894907", + "dotnettests_macos.runTests.TESTS_JOBSTATUS": "Failed", + "dotnettests_ios.fix_commit.GIT_HASH": "fa3d1deb4e2d0ac358f2e0ac80e3d305ca541848", + "dotnettests_ios.DownloadPipelineArtifact1.BuildNumber": "8894907", + "dotnettests_ios.Bash23.TESTS_ATTEMPT": "1", + "dotnettests_ios.Bash23.TESTS_BOT": "XAMMINI-014.Ventura", + "dotnettests_ios.Bash23.TESTS_JOBSTATUS": "Failed", + "dotnettests_ios.Bash23.TESTS_LABEL": "dotnettests", + "dotnettests_ios.Bash23.TESTS_PLATFORM": "", + "dotnettests_ios.Bash23.TESTS_TITLE": "dotnettests_ios", + "dotnettests_ios.DownloadPipelineArtifact2.BuildNumber": "8894907", + "dotnettests_ios.runTests.TESTS_JOBSTATUS": "Succeeded", + "dotnettests_multiple.fix_commit.GIT_HASH": "fa3d1deb4e2d0ac358f2e0ac80e3d305ca541848", + "dotnettests_multiple.DownloadPipelineArtifact1.BuildNumber": "8894907", + "dotnettests_multiple.Bash23.TESTS_ATTEMPT": "1", + "dotnettests_multiple.Bash23.TESTS_BOT": "XAMMINI-010.Ventura", + "dotnettests_multiple.Bash23.TESTS_JOBSTATUS": "Failed", + "dotnettests_multiple.Bash23.TESTS_LABEL": "dotnettests", + "dotnettests_multiple.Bash23.TESTS_PLATFORM": "", + "dotnettests_multiple.Bash23.TESTS_TITLE": "dotnettests_multiple", + "dotnettests_multiple.DownloadPipelineArtifact2.BuildNumber": "8894907", + "dotnettests_multiple.runTests.TESTS_JOBSTATUS": "Succeeded", + "cecil.fix_commit.GIT_HASH": "fa3d1deb4e2d0ac358f2e0ac80e3d305ca541848", + "cecil.DownloadPipelineArtifact1.BuildNumber": "8894907", + "cecil.Bash23.TESTS_ATTEMPT": "1", + "cecil.Bash23.TESTS_BOT": "XAMMINI-013.Ventura", + "cecil.Bash23.TESTS_JOBSTATUS": "Failed", + "cecil.Bash23.TESTS_LABEL": "cecil", + "cecil.Bash23.TESTS_PLATFORM": "", + "cecil.Bash23.TESTS_TITLE": "cecil", + "cecil.DownloadPipelineArtifact2.BuildNumber": "8894907", + "cecil.runTests.TESTS_JOBSTATUS": "Succeeded" + }, + "identifier": null, + "name": "tests", + "attempt": 1, + "startTime": null, + "finishTime": null, + "state": "NotStarted", + "result": "Failed" + } + } +} "@ $stageDependenciesWithMissingResults = @" @@ -942,10 +1031,18 @@ Describe "TestResults tests" { New-Item -Path "$testDirectory/TestSummary-simulator_testsdotnettests_ios-1" -Name "TestSummary.md" -Value "# :tada: All 3 tests passed :tada:" -Force New-Item -Path "$testDirectory/TestSummary-simulator_testsdotnettests_tvos-1" -Name "TestSummary.md" -Value "# :tada: All 4 tests passed :tada:" -Force New-Item -Path "$testDirectory/TestSummary-simulator_testsdotnettests_maccatalyst-1" -Name "TestSummary.md" -Value "5 tests failed, 6 tests passed." -Force - New-Item -Path "$testDirectory/TestSummary-simulator_testsdotnettests_macos-1" -Name "TestSummary.md" -Value "# :tada: All 6 tests passed :tada:" -Force + New-Item -Path "$testDirectory/TestSummary-simulator_testsdotnettests_macos-1" -Name "TestSummary.md" -Value "# Test results + +
+1 tests failed, 4 tests passed. + +## Failed tests + + * Roslyn Transformer tests: TimedOut (Execution timed out after 30 minutes.) +
" -Force New-Item -Path "$testDirectory/TestSummary-simulator_testsdotnettests_multiple-1" -Name "TestSummary.md" -Value "# :tada: All 7 tests passed :tada:" -Force - $parallelResults = New-ParallelTestsResults -Path "$testDirectory" -StageDependencies "$stageDependencies" -Context "context" -VSDropsIndex "vsdropsIndex" + $parallelResults = New-ParallelTestsResults -Path "$testDirectory" -StageDependencies "$stageDependenciesWithMoreFailingTests" -Context "context" -VSDropsIndex "vsdropsIndex" Write-Host "New-ParallelTestsResults return value:" Write-Host $parallelResults @@ -964,7 +1061,7 @@ Describe "TestResults tests" { $content | Should -Be "# Test results :x: Tests failed on context -0 tests crashed, 5 tests failed, 27 tests passed. +0 tests crashed, 6 tests failed, 25 tests passed. ## Failures @@ -976,11 +1073,22 @@ Describe "TestResults tests" { [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_maccatalyst-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_maccatalyst-1&api-version=6.0&`$format=zip) +### :x: dotnettests tests (macOS) + +
+1 tests failed, 4 tests passed. + +## Failed tests + + * Roslyn Transformer tests: TimedOut (Execution timed out after 30 minutes.) +
+ +[Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_macos-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_macos-1&api-version=6.0&`$format=zip) + ## Successes :white_check_mark: cecil: All 1 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagececil-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagececil-1&api-version=6.0&`$format=zip) :white_check_mark: dotnettests (iOS): All 3 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_ios-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_ios-1&api-version=6.0&`$format=zip) -:white_check_mark: dotnettests (macOS): All 6 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_macos-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_macos-1&api-version=6.0&`$format=zip) :white_check_mark: dotnettests (Multiple platforms): All 7 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_multiple-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_multiple-1&api-version=6.0&`$format=zip) :white_check_mark: dotnettests (tvOS): All 4 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_tvos-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_tvos-1&api-version=6.0&`$format=zip) diff --git a/tools/devops/automation/scripts/TestResults.psm1 b/tools/devops/automation/scripts/TestResults.psm1 index 48b19a1d0e3d..1f64cde08f6b 100644 --- a/tools/devops/automation/scripts/TestResults.psm1 +++ b/tools/devops/automation/scripts/TestResults.psm1 @@ -404,17 +404,18 @@ class ParallelTestsResults { $startLine = $i } $addDetails = $false - break } elseif ($line.Contains("")) { if ($startLine -eq -1) { $startLine = $i } $addSummary = $false - break } elseif ($line.Contains("## Failed tests")) { $startLine = $i + 1 break } + if (($addDetails -eq $false) -and ($addSummary -eq $false)) { + break + } } } else { $resultLines = @("Test has no summary file.")