Skip to content

[devops] Fix a few issues with pwsh tests and test result parsing. #24516

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rolfbjarne merged 2 commits into from
Jan 20, 2026
Merged

[devops] Fix a few issues with pwsh tests and test result parsing. #24516

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2ff1b60
[devops] Fix a few issues with pwsh tests and test result parsing.
rolfbjarne Jan 16, 2026
f0dc8ba
More fixes
rolfbjarne Jan 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 11 additions & 15 deletions tools/devops/automation/scripts/GitHub.Tests.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,11 +247,7 @@ Describe 'IsCurrentCommitLatestInPR' {
}
} -ModuleName 'GitHub'

$githubComments = [GitHubComments]::new("testorg", "testrepo", "test-token", "abc123def456")
$githubComments.PRIds = @("123")

$result = $githubComments.IsCurrentCommitLatestInPR()

$result = Get-IsCurrentCommitLatestInPR -Org "testorg" -Repo "testrepo" -Token "test-token" -Hash "abc123def456" -PrIDs @("123")
$result | Should -Be $true
}

Expand All @@ -264,20 +260,20 @@ Describe 'IsCurrentCommitLatestInPR' {
}
} -ModuleName 'GitHub'

$githubComments = [GitHubComments]::new("testorg", "testrepo", "test-token", "abc123def456")
$githubComments.PRIds = @("123")

$result = $githubComments.IsCurrentCommitLatestInPR()

$result = Get-IsCurrentCommitLatestInPR -Org "testorg" -Repo "testrepo" -Token "test-token" -Hash "abc123def456" -PrIDs @("123")
$result | Should -Be $false
}

It 'returns true when not in PR context' {
$githubComments = [GitHubComments]::new("testorg", "testrepo", "test-token", "abc123def456")
$githubComments.PRIds = @() # Empty array means not in PR

$result = $githubComments.IsCurrentCommitLatestInPR()

Mock Invoke-Request {
return @{
"head" = @{
"sha" = "different123hash"
}
}
} -ModuleName 'GitHub'
Comment on lines +268 to +274
Copy link

Copilot AI Jan 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The mock in the test 'returns true when not in PR context' is unnecessary. Since an empty PR IDs array is passed, the IsCurrentCommitLatestInPR method will return true without making any API call, so this mock will never be invoked. The mock should be removed or the test should verify that Invoke-Request is not called.

Copilot uses AI. Check for mistakes.

$result = Get-IsCurrentCommitLatestInPR -Org "testorg" -Repo "testrepo" -Token "test-token" -Hash "abc123def456" -PrIDs @() # Empty array means not in PR
$result | Should -Be $true
}
}
Expand Down
28 changes: 28 additions & 0 deletions tools/devops/automation/scripts/GitHub.psm1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1305,6 +1305,33 @@ function Convert-Markdown {
return $InputContents
}

function Get-IsCurrentCommitLatestInPR {
param (
[ValidateNotNullOrEmpty ()]
[string]
$Org,

[ValidateNotNullOrEmpty ()]
[string]
$Repo,

[ValidateNotNullOrEmpty ()]
[string]
$Token,

[string]
$Hash,

[string[]]
$PrIDs
)

$githubComments = New-GitHubCommentsObject -Org $Org -Repo $Repo -Token $Token -Hash $Hash
$githubComments.PRIds = $PrIDs
$result = $githubComments.IsCurrentCommitLatestInPR()
return $result
}

# module exports, any other functions are private and should not be used outside the module.
Export-ModuleMember -Function New-GitHubComment
Export-ModuleMember -Function Get-GitHubPRInfo
Expand All @@ -1313,6 +1340,7 @@ Export-ModuleMember -Function New-GistWithFiles
Export-ModuleMember -Function New-GistObjectDefinition
Export-ModuleMember -Function New-GistWithContent
Export-ModuleMember -Function Convert-Markdown
Export-ModuleMember -Function Get-IsCurrentCommitLatestInPR

# new future API that uses objects.
Export-ModuleMember -Function New-GitHubCommentsObject
Expand Down
119 changes: 113 additions & 6 deletions tools/devops/automation/scripts/TestResults.Tests.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,95 @@ Describe "TestResults tests" {
}
}
}
"@

$stageDependenciesWithMoreFailingTests = @"
{
"build_macos_tests": {
"build_macos_tests_job": {
"result": "Succeeded"
}
},
"configure_build": {
"configure": {
"outputs": {
"test_matrix.TEST_MATRIX": "$($matrix.Replace("`n", "\n").Replace("`"", "\`""))"
}
}
},
"simulator_tests": {
"tests": {
"outputs": {
"dotnettests_tvos.runTests.TESTS_JOBSTATUS": "Succeeded",
"dotnettests_tvos.fix_commit.GIT_HASH": "fa3d1deb4e2d0ac358f2e0ac80e3d305ca541848",
"dotnettests_tvos.DownloadPipelineArtifact1.BuildNumber": "8894907",
"dotnettests_tvos.Bash23.TESTS_ATTEMPT": "1",
"dotnettests_tvos.Bash23.TESTS_BOT": "XAMMINI-012.Ventura",
"dotnettests_tvos.Bash23.TESTS_JOBSTATUS": "Failed",
"dotnettests_tvos.Bash23.TESTS_LABEL": "dotnettests",
"dotnettests_tvos.Bash23.TESTS_PLATFORM": "",
"dotnettests_tvos.Bash23.TESTS_TITLE": "dotnettests_tvos",
"dotnettests_tvos.DownloadPipelineArtifact2.BuildNumber": "8894907",
"dotnettests_maccatalyst.fix_commit.GIT_HASH": "fa3d1deb4e2d0ac358f2e0ac80e3d305ca541848",
"dotnettests_maccatalyst.DownloadPipelineArtifact1.BuildNumber": "8894907",
"dotnettests_maccatalyst.Bash23.TESTS_ATTEMPT": "1",
"dotnettests_maccatalyst.Bash23.TESTS_BOT": "XAMBOT-1023.Ventura",
"dotnettests_maccatalyst.Bash23.TESTS_JOBSTATUS": "Failed",
"dotnettests_maccatalyst.Bash23.TESTS_LABEL": "dotnettests",
"dotnettests_maccatalyst.Bash23.TESTS_PLATFORM": "",
"dotnettests_maccatalyst.Bash23.TESTS_TITLE": "dotnettests_maccatalyst",
"dotnettests_maccatalyst.DownloadPipelineArtifact2.BuildNumber": "8894907",
"dotnettests_macos.fix_commit.GIT_HASH": "fa3d1deb4e2d0ac358f2e0ac80e3d305ca541848",
"dotnettests_macos.DownloadPipelineArtifact1.BuildNumber": "8894907",
"dotnettests_macos.Bash23.TESTS_ATTEMPT": "1",
"dotnettests_macos.Bash23.TESTS_BOT": "XAMMINI-015.Ventura",
"dotnettests_macos.Bash23.TESTS_JOBSTATUS": "Failed",
"dotnettests_macos.Bash23.TESTS_LABEL": "dotnettests",
"dotnettests_macos.Bash23.TESTS_PLATFORM": "",
"dotnettests_macos.Bash23.TESTS_TITLE": "dotnettests_macos",
"dotnettests_macos.DownloadPipelineArtifact2.BuildNumber": "8894907",
"dotnettests_macos.runTests.TESTS_JOBSTATUS": "Failed",
"dotnettests_ios.fix_commit.GIT_HASH": "fa3d1deb4e2d0ac358f2e0ac80e3d305ca541848",
"dotnettests_ios.DownloadPipelineArtifact1.BuildNumber": "8894907",
"dotnettests_ios.Bash23.TESTS_ATTEMPT": "1",
"dotnettests_ios.Bash23.TESTS_BOT": "XAMMINI-014.Ventura",
"dotnettests_ios.Bash23.TESTS_JOBSTATUS": "Failed",
"dotnettests_ios.Bash23.TESTS_LABEL": "dotnettests",
"dotnettests_ios.Bash23.TESTS_PLATFORM": "",
"dotnettests_ios.Bash23.TESTS_TITLE": "dotnettests_ios",
"dotnettests_ios.DownloadPipelineArtifact2.BuildNumber": "8894907",
"dotnettests_ios.runTests.TESTS_JOBSTATUS": "Succeeded",
"dotnettests_multiple.fix_commit.GIT_HASH": "fa3d1deb4e2d0ac358f2e0ac80e3d305ca541848",
"dotnettests_multiple.DownloadPipelineArtifact1.BuildNumber": "8894907",
"dotnettests_multiple.Bash23.TESTS_ATTEMPT": "1",
"dotnettests_multiple.Bash23.TESTS_BOT": "XAMMINI-010.Ventura",
"dotnettests_multiple.Bash23.TESTS_JOBSTATUS": "Failed",
"dotnettests_multiple.Bash23.TESTS_LABEL": "dotnettests",
"dotnettests_multiple.Bash23.TESTS_PLATFORM": "",
"dotnettests_multiple.Bash23.TESTS_TITLE": "dotnettests_multiple",
"dotnettests_multiple.DownloadPipelineArtifact2.BuildNumber": "8894907",
"dotnettests_multiple.runTests.TESTS_JOBSTATUS": "Succeeded",
"cecil.fix_commit.GIT_HASH": "fa3d1deb4e2d0ac358f2e0ac80e3d305ca541848",
"cecil.DownloadPipelineArtifact1.BuildNumber": "8894907",
"cecil.Bash23.TESTS_ATTEMPT": "1",
"cecil.Bash23.TESTS_BOT": "XAMMINI-013.Ventura",
"cecil.Bash23.TESTS_JOBSTATUS": "Failed",
"cecil.Bash23.TESTS_LABEL": "cecil",
"cecil.Bash23.TESTS_PLATFORM": "",
"cecil.Bash23.TESTS_TITLE": "cecil",
"cecil.DownloadPipelineArtifact2.BuildNumber": "8894907",
"cecil.runTests.TESTS_JOBSTATUS": "Succeeded"
},
"identifier": null,
"name": "tests",
"attempt": 1,
"startTime": null,
"finishTime": null,
"state": "NotStarted",
"result": "Failed"
}
}
}
"@

$stageDependenciesWithMissingResults = @"
Expand Down Expand Up @@ -915,7 +1004,7 @@ Describe "TestResults tests" {

<summary>1 tests failed, 0 tests passed.</summary>
<details>

# :tada: All 5 tests passed :tada:
</details>

[Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_maccatalyst-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_maccatalyst-1&api-version=6.0&`$format=zip)
Expand All @@ -942,10 +1031,18 @@ Describe "TestResults tests" {
New-Item -Path "$testDirectory/TestSummary-simulator_testsdotnettests_ios-1" -Name "TestSummary.md" -Value "# :tada: All 3 tests passed :tada:" -Force
New-Item -Path "$testDirectory/TestSummary-simulator_testsdotnettests_tvos-1" -Name "TestSummary.md" -Value "# :tada: All 4 tests passed :tada:" -Force
New-Item -Path "$testDirectory/TestSummary-simulator_testsdotnettests_maccatalyst-1" -Name "TestSummary.md" -Value "<summary>5 tests failed, 6 tests passed.</summary>" -Force
New-Item -Path "$testDirectory/TestSummary-simulator_testsdotnettests_macos-1" -Name "TestSummary.md" -Value "# :tada: All 6 tests passed :tada:" -Force
New-Item -Path "$testDirectory/TestSummary-simulator_testsdotnettests_macos-1" -Name "TestSummary.md" -Value "# Test results

<details>
<summary>1 tests failed, 4 tests passed.</summary>

## Failed tests

* Roslyn Transformer tests: TimedOut (Execution timed out after 30 minutes.)
</details>" -Force
New-Item -Path "$testDirectory/TestSummary-simulator_testsdotnettests_multiple-1" -Name "TestSummary.md" -Value "# :tada: All 7 tests passed :tada:" -Force

$parallelResults = New-ParallelTestsResults -Path "$testDirectory" -StageDependencies "$stageDependencies" -Context "context" -VSDropsIndex "vsdropsIndex"
$parallelResults = New-ParallelTestsResults -Path "$testDirectory" -StageDependencies "$stageDependenciesWithMoreFailingTests" -Context "context" -VSDropsIndex "vsdropsIndex"

Write-Host "New-ParallelTestsResults return value:"
Write-Host $parallelResults
Expand All @@ -964,24 +1061,34 @@ Describe "TestResults tests" {
$content | Should -Be "# Test results
:x: Tests failed on context

0 tests crashed, 5 tests failed, 27 tests passed.
0 tests crashed, 6 tests failed, 25 tests passed.

## Failures

### :x: dotnettests tests (MacCatalyst)

<details>
<summary>5 tests failed, 6 tests passed.</summary>
</details>

[Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_maccatalyst-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_maccatalyst-1&api-version=6.0&`$format=zip)

### :x: dotnettests tests (macOS)

<details>
<summary>1 tests failed, 4 tests passed.</summary>

## Failed tests

* Roslyn Transformer tests: TimedOut (Execution timed out after 30 minutes.)
</details>

[Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_maccatalyst-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_maccatalyst-1&api-version=6.0&`$format=zip)
[Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_macos-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_macos-1&api-version=6.0&`$format=zip)

## Successes

:white_check_mark: cecil: All 1 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagececil-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagececil-1&api-version=6.0&`$format=zip)
:white_check_mark: dotnettests (iOS): All 3 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_ios-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_ios-1&api-version=6.0&`$format=zip)
:white_check_mark: dotnettests (macOS): All 6 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_macos-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_macos-1&api-version=6.0&`$format=zip)
:white_check_mark: dotnettests (Multiple platforms): All 7 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_multiple-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_multiple-1&api-version=6.0&`$format=zip)
:white_check_mark: dotnettests (tvOS): All 4 tests passed. [Html Report (VSDrops)](vsdropsIndex/testStagedotnettests_tvos-1/;/tests/vsdrops_index.html) [Download](/_apis/build/builds//artifacts?artifactName=HtmlReport-testStagedotnettests_tvos-1&api-version=6.0&`$format=zip)

Expand Down
25 changes: 20 additions & 5 deletions tools/devops/automation/scripts/TestResults.psm1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,32 +393,47 @@ class ParallelTestsResults {
$stringBuilder.AppendLine("")
} else {
$addSummary = $true
$startLine = 0
$addDetails = $true
$startLine = -1
if (Test-Path -Path $r.ResultsPath -PathType Leaf) {
$resultLines = Get-Content -Path $r.ResultsPath
$resultLines = @(Get-Content -Path $r.ResultsPath)
for ($i = 0; $i -lt $resultLines.Length; $i++) {
$line = $resultLines[$i]
if ($line.Contains("<details>")) {
$startLine = $i
if ($startLine -eq -1) {
$startLine = $i
}
$addDetails = $false
} elseif ($line.Contains("<summary>")) {
if ($startLine -eq -1) {
$startLine = $i
}
$addSummary = $false
break
} elseif ($line.Contains("## Failed tests")) {
$startLine = $i + 1
break
}
if (($addDetails -eq $false) -and ($addSummary -eq $false)) {
break
}
}
} else {
$resultLines = @("Test has no summary file.")
}

if ($addSummary) {
$stringBuilder.AppendLine("<summary>$($result.Failed) tests failed, $($result.Passed) tests passed.</summary>")
}
if ($addDetails) {
$stringBuilder.AppendLine("<details>")
}
if ($startLine -eq -1) {
$startLine = 0
}
for ($i = $startLine; $i -lt $resultLines.Length; $i++) {
$stringBuilder.AppendLine($resultLines[$i])
}
if ($addSummary) {
if ($addDetails) {
$stringBuilder.AppendLine("</details>")
}

Expand Down
Loading