[devops] Show "(Publish failed)" instead of broken VSDrops link when html report publish fails.

tools/devops/automation/scripts/TestResults.Tests.ps1

@@ -1352,4 +1352,76 @@ Test results reported success, but the tests job failed.
             $sonomaIdx | Should -BeLessThan $sequoiaIdx
         }
     }
+
+    Context "VSDrops publish failed" {
+        It "shows publish failed text instead of VSDrops link" {
+            $VerbosePreference = "Continue"
+            $DebugPreference = "Continue"
+
+            $vsdropsMatrix = @"
+{
+    "cecil": {
+        "LABEL": "cecil",
+        "TESTS_LABELS": "--label=skip-all-tests,run-cecil-tests",
+        "TEST_STAGE": "simulator_tests",
+        "LABEL_WITH_PLATFORM": "cecil",
+        "STATUS_CONTEXT": "VSTS: simulator tests - cecil",
+        "TEST_PREFIX": "simulator_testscecil",
+        "TEST_PLATFORM": ""
+    }
+}
+"@
+            $vsdropsFailedStageDeps = @"
+{
+  "configure_build": {
+    "configure": {
+      "outputs": {
+        "test_matrix.TEST_MATRIX": "$($vsdropsMatrix.Replace("`n", "\n").Replace("`"", "\`""))"
+      }
+    }
+  },
+  "simulator_tests": {
+    "tests": {
+      "outputs": {
+        "cecil.PowerShell15.TESTS_ATTEMPT": "1",
+        "cecil.PowerShell15.TESTS_BOT": "XAMMINI-013.Ventura",
+        "cecil.PowerShell15.TESTS_LABEL": "cecil",
+        "cecil.PowerShell15.TESTS_PLATFORM": "",
+        "cecil.PowerShell15.TESTS_TITLE": "cecil",
+        "cecil.runTests.TESTS_JOBSTATUS": "Succeeded",
+        "cecil.setVSDropsPublishResult.VSDROPS_PUBLISHED": "Failed"
+      },
+      "identifier": null,
+      "name": "tests",
+      "attempt": 1,
+      "startTime": null,
+      "finishTime": null,
+      "state": "NotStarted",
+      "result": "Succeeded"
+    }
+  }
+}
+"@
+            $testDirectory = Join-Path "." "subdir"
+            New-Item -Path "$testDirectory" -ItemType "directory" -Force
+            New-Item -Path "$testDirectory/TestSummary-simulator_testscecil-1" -Name "TestSummary.md" -Value "# :tada: All 1 tests passed :tada:" -Force
+
+            $parallelResults = New-ParallelTestsResults -Path "$testDirectory" -StageDependencies "$vsdropsFailedStageDeps" -Context "context" -VSDropsIndex "vsdropsIndex"
+
+            $parallelResults.IsSuccess() | Should -Be $true
+
+            $sb = [System.Text.StringBuilder]::new()
+            $parallelResults.WriteComment($sb)
+
+            Remove-Item -Path $testDirectory -Recurse
+
+            $content = $sb.ToString()
+
+            Write-Host $content
+
+            $content | Should -Not -BeLike "*[Html Report (VSDrops)]*"
+            $content | Should -BeLike "*(:warning: Html Report Publish failed :warning:)*"
+            $content | Should -BeLike "*[Download]*"
+        }
+    }
 }

tools/devops/automation/scripts/TestResults.psm1

@@ -61,6 +61,7 @@ class TestResult {
     [string] $TestStage
     [string] $DisplayName
     [bool] $IsMacTest
+    [bool] $VSDropsPublishFailed
     hidden [int] $Passed
     hidden [int] $Failed
     hidden [string[]] $NotTestSummaryLabels = @()
@@ -339,9 +340,13 @@ class ParallelTestsResults {
     }
 
     [string] GetDownloadLinks($testResult) {
-        $dropsIndex = "$($this.VSDropsIndex)/$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)/;/tests/vsdrops_index.html"
         $artifactUrl = "$Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI$Env:SYSTEM_TEAMPROJECT/_apis/build/builds/$Env:BUILD_BUILDID/artifacts?artifactName=HtmlReport-$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)&api-version=6.0&`$format=zip"
-        $downloadInfo = "[Html Report (VSDrops)]($dropsIndex) [Download]($artifactUrl)"
+        if ($testResult.VSDropsPublishFailed) {
+            $downloadInfo = "(:warning: Html Report Publish failed :warning:) [Download]($artifactUrl)"
+        } else {
+            $dropsIndex = "$($this.VSDropsIndex)/$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)/;/tests/vsdrops_index.html"
+            $downloadInfo = "[Html Report (VSDrops)]($dropsIndex) [Download]($artifactUrl)"
+        }
         return $downloadInfo
     }
 
@@ -589,6 +594,7 @@ class ParallelTestsResults {
                         $platformKey = $outputs.Keys | Where-Object { $_.EndsWith(".TESTS_PLATFORM") }
                         $attemptKey = $outputs.Keys | Where-Object { $_.EndsWith(".TESTS_ATTEMPT") }
                         $titleKey = $outputs.Keys | Where-Object { $_.EndsWith(".TESTS_TITLE") }
+                        $vsdropsPublishedKey = $outputs.Keys | Where-Object { $_.EndsWith(".VSDROPS_PUBLISHED") } | Sort-Object | Select-Object -Last 1
                     } else {
                         # matrix job
                         $jobName = $name.Substring(0, $name.IndexOf('.'))
@@ -597,6 +603,7 @@ class ParallelTestsResults {
                         $platformKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".TESTS_PLATFORM") }
                         $attemptKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".TESTS_ATTEMPT") }
                         $titleKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".TESTS_TITLE") }
+                        $vsdropsPublishedKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".VSDROPS_PUBLISHED") } | Sort-Object | Select-Object -Last 1
                     }
 
                     Write-Host "Keys for Label='$label' and JobName='$jobName' (dotCount=$dotCount): TitleKey='$titleKey'  StatusKey=$statusKey BotKey=$botKey PlatformKey=$platformKey AttemptKey=$attemptKey"
@@ -611,6 +618,7 @@ class ParallelTestsResults {
                     $platform = if ($platformKey -eq $null) { "NotFound" } else { $outputs[$platformKey] }
                     $attempt = if ($attemptKey -eq $null) { -2 } else { [int]$outputs[$attemptKey] }
                     $title = if ($titleKey -eq $null) { "NotFound" } else { $outputs[$titleKey] }
+                    $vsdropsPublished = if ($vsdropsPublishedKey -eq $null) { $null } else { $outputs[$vsdropsPublishedKey] }
                     $testResult = [PSCustomObject]@{
                         Label = $label
                         Title = $title
@@ -619,6 +627,7 @@ class ParallelTestsResults {
                         Platform = $platform
                         Attempt = $attempt
                         TestStage = $testStage
+                        VSDropsPublished = $vsdropsPublished
                     }
                     if ($tests.Contains($label)) {
                         $testInfo = $tests[$label]
@@ -675,6 +684,7 @@ class ParallelTestsResults {
                     }
 
                     $result = [TestResult]::new($testSummaryPath, $status, $testConfig, $testAttempt)
+                    $result.VSDropsPublishFailed = ($testResult.VSDropsPublished -eq "Failed")
                 }
 
                 $testResults += $result

tools/devops/automation/templates/mac/build.yml

@@ -210,8 +210,15 @@ steps:
   condition: succeededOrFailed()
 
 # Upload to VSDrops so that the Html Report link in the GitHub comment works.
+- bash: |
+    echo "##vso[task.setvariable variable=JOB_STATUS_BEFORE_VSDROPS]$AGENT_JOBSTATUS"
+  displayName: 'Capture job status before VSDrops publish'
+  condition: succeededOrFailed()
+  continueOnError: true
+
 - task: artifactDropTask@1
   displayName: 'Publish to Artifact Services Drop'
+  name: publishVSDrops
   inputs:
     dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection'
     dropMetadataContainerName: '${{ parameters.uploadPrefix }}DropMetadata-${{ parameters.stageName }}${{ parameters.label }}-$(System.JobAttempt)'
@@ -223,6 +230,19 @@ steps:
   continueOnError: true
   condition: succeededOrFailed()
 
+# Azure DevOps doesn't support checking individual step results in step conditions
+# (failed('stepName') only works for jobs/stages). Comparing AGENT_JOBSTATUS before
+# and after the publish step is the best available approach to detect publish failures.
+- bash: |
+    if [ "$JOB_STATUS_BEFORE_VSDROPS" != "$AGENT_JOBSTATUS" ]; then
+      echo "VSDrops publish changed job status from '$JOB_STATUS_BEFORE_VSDROPS' to '$AGENT_JOBSTATUS'"
+      echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed"
+    fi
+  name: setVSDropsPublishResult
+  displayName: 'Set VSDrops publish result'
+  continueOnError: true
+  condition: succeededOrFailed()
+
 # Archive files for the Html Report so that the report can be easily uploaded as artifacts of the build.
 - task: ArchiveFiles@1
   displayName: 'Archive HtmlReport'

tools/devops/automation/templates/tests/run-tests.yml

@@ -182,8 +182,15 @@ steps:
   continueOnError: true
   condition: succeededOrFailed()
 
+- bash: |
+    echo "##vso[task.setvariable variable=JOB_STATUS_BEFORE_VSDROPS]$AGENT_JOBSTATUS"
+  displayName: 'Capture job status before VSDrops publish'
+  condition: succeededOrFailed()
+  continueOnError: true
+
 - task: artifactDropTask@1
   displayName: 'Publish to Artifact Services Drop'
+  name: publishVSDrops
   inputs:
     dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection'
     dropMetadataContainerName: '${{ parameters.uploadPrefix }}DropMetadata-${{ parameters.testPrefix }}${{ parameters.labelWithPlatform }}-$(System.JobAttempt)'
@@ -195,6 +202,19 @@ steps:
   continueOnError: true
   condition: succeededOrFailed()
 
+# Azure DevOps doesn't support checking individual step results in step conditions
+# (failed('stepName') only works for jobs/stages). Comparing AGENT_JOBSTATUS before
+# and after the publish step is the best available approach to detect publish failures.
+- bash: |
+    if [ "$JOB_STATUS_BEFORE_VSDROPS" != "$AGENT_JOBSTATUS" ]; then
+      echo "VSDrops publish changed job status from '$JOB_STATUS_BEFORE_VSDROPS' to '$AGENT_JOBSTATUS'"
+      echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed"
+    fi
+  name: setVSDropsPublishResult
+  displayName: 'Set VSDrops publish result'
+  continueOnError: true
+  condition: succeededOrFailed()
+
 - bash: |
     set -ex
     find . -name 'vsts-*.xml' || true