diff --git a/tools/devops/automation/scripts/TestResults.Tests.ps1 b/tools/devops/automation/scripts/TestResults.Tests.ps1 index 1a27744780c..0d29999bc54 100644 --- a/tools/devops/automation/scripts/TestResults.Tests.ps1 +++ b/tools/devops/automation/scripts/TestResults.Tests.ps1 @@ -1352,4 +1352,76 @@ Test results reported success, but the tests job failed. $sonomaIdx | Should -BeLessThan $sequoiaIdx } } + + Context "VSDrops publish failed" { + It "shows publish failed text instead of VSDrops link" { + $VerbosePreference = "Continue" + $DebugPreference = "Continue" + + $vsdropsMatrix = @" +{ + "cecil": { + "LABEL": "cecil", + "TESTS_LABELS": "--label=skip-all-tests,run-cecil-tests", + "TEST_STAGE": "simulator_tests", + "LABEL_WITH_PLATFORM": "cecil", + "STATUS_CONTEXT": "VSTS: simulator tests - cecil", + "TEST_PREFIX": "simulator_testscecil", + "TEST_PLATFORM": "" + } +} +"@ + $vsdropsFailedStageDeps = @" +{ + "configure_build": { + "configure": { + "outputs": { + "test_matrix.TEST_MATRIX": "$($vsdropsMatrix.Replace("`n", "\n").Replace("`"", "\`""))" + } + } + }, + "simulator_tests": { + "tests": { + "outputs": { + "cecil.PowerShell15.TESTS_ATTEMPT": "1", + "cecil.PowerShell15.TESTS_BOT": "XAMMINI-013.Ventura", + "cecil.PowerShell15.TESTS_LABEL": "cecil", + "cecil.PowerShell15.TESTS_PLATFORM": "", + "cecil.PowerShell15.TESTS_TITLE": "cecil", + "cecil.runTests.TESTS_JOBSTATUS": "Succeeded", + "cecil.setVSDropsPublishResult.VSDROPS_PUBLISHED": "Failed" + }, + "identifier": null, + "name": "tests", + "attempt": 1, + "startTime": null, + "finishTime": null, + "state": "NotStarted", + "result": "Succeeded" + } + } +} +"@ + $testDirectory = Join-Path "." "subdir" + New-Item -Path "$testDirectory" -ItemType "directory" -Force + New-Item -Path "$testDirectory/TestSummary-simulator_testscecil-1" -Name "TestSummary.md" -Value "# :tada: All 1 tests passed :tada:" -Force + + $parallelResults = New-ParallelTestsResults -Path "$testDirectory" -StageDependencies "$vsdropsFailedStageDeps" -Context "context" -VSDropsIndex "vsdropsIndex" + + $parallelResults.IsSuccess() | Should -Be $true + + $sb = [System.Text.StringBuilder]::new() + $parallelResults.WriteComment($sb) + + Remove-Item -Path $testDirectory -Recurse + + $content = $sb.ToString() + + Write-Host $content + + $content | Should -Not -BeLike "*[Html Report (VSDrops)]*" + $content | Should -BeLike "*(:warning: Html Report Publish failed :warning:)*" + $content | Should -BeLike "*[Download]*" + } + } } diff --git a/tools/devops/automation/scripts/TestResults.psm1 b/tools/devops/automation/scripts/TestResults.psm1 index ef5b0eb0645..da6b71d7107 100644 --- a/tools/devops/automation/scripts/TestResults.psm1 +++ b/tools/devops/automation/scripts/TestResults.psm1 @@ -61,6 +61,7 @@ class TestResult { [string] $TestStage [string] $DisplayName [bool] $IsMacTest + [bool] $VSDropsPublishFailed hidden [int] $Passed hidden [int] $Failed hidden [string[]] $NotTestSummaryLabels = @() @@ -339,9 +340,13 @@ class ParallelTestsResults { } [string] GetDownloadLinks($testResult) { - $dropsIndex = "$($this.VSDropsIndex)/$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)/;/tests/vsdrops_index.html" $artifactUrl = "$Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI$Env:SYSTEM_TEAMPROJECT/_apis/build/builds/$Env:BUILD_BUILDID/artifacts?artifactName=HtmlReport-$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)&api-version=6.0&`$format=zip" - $downloadInfo = "[Html Report (VSDrops)]($dropsIndex) [Download]($artifactUrl)" + if ($testResult.VSDropsPublishFailed) { + $downloadInfo = "(:warning: Html Report Publish failed :warning:) [Download]($artifactUrl)" + } else { + $dropsIndex = "$($this.VSDropsIndex)/$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)/;/tests/vsdrops_index.html" + $downloadInfo = "[Html Report (VSDrops)]($dropsIndex) [Download]($artifactUrl)" + } return $downloadInfo } @@ -589,6 +594,7 @@ class ParallelTestsResults { $platformKey = $outputs.Keys | Where-Object { $_.EndsWith(".TESTS_PLATFORM") } $attemptKey = $outputs.Keys | Where-Object { $_.EndsWith(".TESTS_ATTEMPT") } $titleKey = $outputs.Keys | Where-Object { $_.EndsWith(".TESTS_TITLE") } + $vsdropsPublishedKey = $outputs.Keys | Where-Object { $_.EndsWith(".VSDROPS_PUBLISHED") } | Sort-Object | Select-Object -Last 1 } else { # matrix job $jobName = $name.Substring(0, $name.IndexOf('.')) @@ -597,6 +603,7 @@ class ParallelTestsResults { $platformKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".TESTS_PLATFORM") } $attemptKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".TESTS_ATTEMPT") } $titleKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".TESTS_TITLE") } + $vsdropsPublishedKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".VSDROPS_PUBLISHED") } | Sort-Object | Select-Object -Last 1 } Write-Host "Keys for Label='$label' and JobName='$jobName' (dotCount=$dotCount): TitleKey='$titleKey' StatusKey=$statusKey BotKey=$botKey PlatformKey=$platformKey AttemptKey=$attemptKey" @@ -611,6 +618,7 @@ class ParallelTestsResults { $platform = if ($platformKey -eq $null) { "NotFound" } else { $outputs[$platformKey] } $attempt = if ($attemptKey -eq $null) { -2 } else { [int]$outputs[$attemptKey] } $title = if ($titleKey -eq $null) { "NotFound" } else { $outputs[$titleKey] } + $vsdropsPublished = if ($vsdropsPublishedKey -eq $null) { $null } else { $outputs[$vsdropsPublishedKey] } $testResult = [PSCustomObject]@{ Label = $label Title = $title @@ -619,6 +627,7 @@ class ParallelTestsResults { Platform = $platform Attempt = $attempt TestStage = $testStage + VSDropsPublished = $vsdropsPublished } if ($tests.Contains($label)) { $testInfo = $tests[$label] @@ -675,6 +684,7 @@ class ParallelTestsResults { } $result = [TestResult]::new($testSummaryPath, $status, $testConfig, $testAttempt) + $result.VSDropsPublishFailed = ($testResult.VSDropsPublished -eq "Failed") } $testResults += $result diff --git a/tools/devops/automation/templates/mac/build.yml b/tools/devops/automation/templates/mac/build.yml index b564d24af9a..770a2358fb2 100644 --- a/tools/devops/automation/templates/mac/build.yml +++ b/tools/devops/automation/templates/mac/build.yml @@ -210,8 +210,15 @@ steps: condition: succeededOrFailed() # Upload to VSDrops so that the Html Report link in the GitHub comment works. +- bash: | + echo "##vso[task.setvariable variable=JOB_STATUS_BEFORE_VSDROPS]$AGENT_JOBSTATUS" + displayName: 'Capture job status before VSDrops publish' + condition: succeededOrFailed() + continueOnError: true + - task: artifactDropTask@1 displayName: 'Publish to Artifact Services Drop' + name: publishVSDrops inputs: dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection' dropMetadataContainerName: '${{ parameters.uploadPrefix }}DropMetadata-${{ parameters.stageName }}${{ parameters.label }}-$(System.JobAttempt)' @@ -223,6 +230,19 @@ steps: continueOnError: true condition: succeededOrFailed() +# Azure DevOps doesn't support checking individual step results in step conditions +# (failed('stepName') only works for jobs/stages). Comparing AGENT_JOBSTATUS before +# and after the publish step is the best available approach to detect publish failures. +- bash: | + if [ "$JOB_STATUS_BEFORE_VSDROPS" != "$AGENT_JOBSTATUS" ]; then + echo "VSDrops publish changed job status from '$JOB_STATUS_BEFORE_VSDROPS' to '$AGENT_JOBSTATUS'" + echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed" + fi + name: setVSDropsPublishResult + displayName: 'Set VSDrops publish result' + continueOnError: true + condition: succeededOrFailed() + - template: ../common/archive-html-report.yml parameters: rootFolder: '$(BUILD_REPOSITORY_TITLE)/jenkins-results' diff --git a/tools/devops/automation/templates/tests/run-tests.yml b/tools/devops/automation/templates/tests/run-tests.yml index 0ff0d43d39c..188474aa28e 100644 --- a/tools/devops/automation/templates/tests/run-tests.yml +++ b/tools/devops/automation/templates/tests/run-tests.yml @@ -182,8 +182,15 @@ steps: continueOnError: true condition: succeededOrFailed() +- bash: | + echo "##vso[task.setvariable variable=JOB_STATUS_BEFORE_VSDROPS]$AGENT_JOBSTATUS" + displayName: 'Capture job status before VSDrops publish' + condition: succeededOrFailed() + continueOnError: true + - task: artifactDropTask@1 displayName: 'Publish to Artifact Services Drop' + name: publishVSDrops inputs: dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection' dropMetadataContainerName: '${{ parameters.uploadPrefix }}DropMetadata-${{ parameters.testPrefix }}${{ parameters.labelWithPlatform }}-$(System.JobAttempt)' @@ -195,6 +202,19 @@ steps: continueOnError: true condition: succeededOrFailed() +# Azure DevOps doesn't support checking individual step results in step conditions +# (failed('stepName') only works for jobs/stages). Comparing AGENT_JOBSTATUS before +# and after the publish step is the best available approach to detect publish failures. +- bash: | + if [ "$JOB_STATUS_BEFORE_VSDROPS" != "$AGENT_JOBSTATUS" ]; then + echo "VSDrops publish changed job status from '$JOB_STATUS_BEFORE_VSDROPS' to '$AGENT_JOBSTATUS'" + echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed" + fi + name: setVSDropsPublishResult + displayName: 'Set VSDrops publish result' + continueOnError: true + condition: succeededOrFailed() + - bash: | set -ex find . -name 'vsts-*.xml' || true