From 16f94b40a73642dd8bc2ccbe9d817d9bdee428db Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Wed, 22 Apr 2026 20:03:37 +0200 Subject: [PATCH 1/3] [devops] Show "(Publish failed)" instead of broken VSDrops link when html report publish fails. When the "Publish to Artifact Services Drop" step times out or fails, the VSDrops link in the GitHub comment would point to a non-existent page. Now detect the failure by setting an output variable when the step does not succeed, and show "(Publish failed)" in plain text instead of a broken link. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../devops/automation/scripts/TestResults.psm1 | 14 ++++++++++++-- tools/devops/automation/templates/mac/build.yml | 17 +++++++++++++++++ .../automation/templates/tests/run-tests.yml | 17 +++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/tools/devops/automation/scripts/TestResults.psm1 b/tools/devops/automation/scripts/TestResults.psm1 index ef5b0eb06457..aab88e9952f3 100644 --- a/tools/devops/automation/scripts/TestResults.psm1 +++ b/tools/devops/automation/scripts/TestResults.psm1 @@ -61,6 +61,7 @@ class TestResult { [string] $TestStage [string] $DisplayName [bool] $IsMacTest + [bool] $VSDropsPublishFailed hidden [int] $Passed hidden [int] $Failed hidden [string[]] $NotTestSummaryLabels = @() @@ -339,9 +340,13 @@ class ParallelTestsResults { } [string] GetDownloadLinks($testResult) { - $dropsIndex = "$($this.VSDropsIndex)/$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)/;/tests/vsdrops_index.html" $artifactUrl = "$Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI$Env:SYSTEM_TEAMPROJECT/_apis/build/builds/$Env:BUILD_BUILDID/artifacts?artifactName=HtmlReport-$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)&api-version=6.0&`$format=zip" - $downloadInfo = "[Html Report (VSDrops)]($dropsIndex) [Download]($artifactUrl)" + if ($testResult.VSDropsPublishFailed) { + $downloadInfo = "(Publish failed) [Download]($artifactUrl)" + } else { + $dropsIndex = "$($this.VSDropsIndex)/$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)/;/tests/vsdrops_index.html" + $downloadInfo = "[Html Report (VSDrops)]($dropsIndex) [Download]($artifactUrl)" + } return $downloadInfo } @@ -589,6 +594,7 @@ class ParallelTestsResults { $platformKey = $outputs.Keys | Where-Object { $_.EndsWith(".TESTS_PLATFORM") } $attemptKey = $outputs.Keys | Where-Object { $_.EndsWith(".TESTS_ATTEMPT") } $titleKey = $outputs.Keys | Where-Object { $_.EndsWith(".TESTS_TITLE") } + $vsdropsPublishedKey = $outputs.Keys | Where-Object { $_.EndsWith(".VSDROPS_PUBLISHED") } } else { # matrix job $jobName = $name.Substring(0, $name.IndexOf('.')) @@ -597,6 +603,7 @@ class ParallelTestsResults { $platformKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".TESTS_PLATFORM") } $attemptKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".TESTS_ATTEMPT") } $titleKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".TESTS_TITLE") } + $vsdropsPublishedKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".VSDROPS_PUBLISHED") } } Write-Host "Keys for Label='$label' and JobName='$jobName' (dotCount=$dotCount): TitleKey='$titleKey' StatusKey=$statusKey BotKey=$botKey PlatformKey=$platformKey AttemptKey=$attemptKey" @@ -611,6 +618,7 @@ class ParallelTestsResults { $platform = if ($platformKey -eq $null) { "NotFound" } else { $outputs[$platformKey] } $attempt = if ($attemptKey -eq $null) { -2 } else { [int]$outputs[$attemptKey] } $title = if ($titleKey -eq $null) { "NotFound" } else { $outputs[$titleKey] } + $vsdropsPublished = if ($vsdropsPublishedKey -eq $null) { $null } else { $outputs[$vsdropsPublishedKey] } $testResult = [PSCustomObject]@{ Label = $label Title = $title @@ -619,6 +627,7 @@ class ParallelTestsResults { Platform = $platform Attempt = $attempt TestStage = $testStage + VSDropsPublished = $vsdropsPublished } if ($tests.Contains($label)) { $testInfo = $tests[$label] @@ -675,6 +684,7 @@ class ParallelTestsResults { } $result = [TestResult]::new($testSummaryPath, $status, $testConfig, $testAttempt) + $result.VSDropsPublishFailed = ($testResult.VSDropsPublished -eq "Failed") } $testResults += $result diff --git a/tools/devops/automation/templates/mac/build.yml b/tools/devops/automation/templates/mac/build.yml index a6c20308996f..f1db621a6e39 100644 --- a/tools/devops/automation/templates/mac/build.yml +++ b/tools/devops/automation/templates/mac/build.yml @@ -210,8 +210,15 @@ steps: condition: succeededOrFailed() # Upload to VSDrops so that the Html Report link in the GitHub comment works. +- bash: | + echo "##vso[task.setvariable variable=JOB_STATUS_BEFORE_VSDROPS]$AGENT_JOBSTATUS" + displayName: 'Capture job status before VSDrops publish' + condition: succeededOrFailed() + continueOnError: true + - task: artifactDropTask@1 displayName: 'Publish to Artifact Services Drop' + name: publishVSDrops inputs: dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection' dropMetadataContainerName: '${{ parameters.uploadPrefix }}DropMetadata-${{ parameters.stageName }}${{ parameters.label }}-$(System.JobAttempt)' @@ -223,6 +230,16 @@ steps: continueOnError: true condition: succeededOrFailed() +- bash: | + if [ "$JOB_STATUS_BEFORE_VSDROPS" != "$AGENT_JOBSTATUS" ]; then + echo "VSDrops publish changed job status from '$JOB_STATUS_BEFORE_VSDROPS' to '$AGENT_JOBSTATUS'" + echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed" + fi + name: setVSDropsPublishResult + displayName: 'Set VSDrops publish result' + continueOnError: true + condition: succeededOrFailed() + # Archive files for the Html Report so that the report can be easily uploaded as artifacts of the build. - task: ArchiveFiles@1 displayName: 'Archive HtmlReport' diff --git a/tools/devops/automation/templates/tests/run-tests.yml b/tools/devops/automation/templates/tests/run-tests.yml index 55f5d8d5a7be..a7b123644b79 100644 --- a/tools/devops/automation/templates/tests/run-tests.yml +++ b/tools/devops/automation/templates/tests/run-tests.yml @@ -182,8 +182,15 @@ steps: continueOnError: true condition: succeededOrFailed() +- bash: | + echo "##vso[task.setvariable variable=JOB_STATUS_BEFORE_VSDROPS]$AGENT_JOBSTATUS" + displayName: 'Capture job status before VSDrops publish' + condition: succeededOrFailed() + continueOnError: true + - task: artifactDropTask@1 displayName: 'Publish to Artifact Services Drop' + name: publishVSDrops inputs: dropServiceURI: 'https://devdiv.artifacts.visualstudio.com/DefaultCollection' dropMetadataContainerName: '${{ parameters.uploadPrefix }}DropMetadata-${{ parameters.testPrefix }}${{ parameters.labelWithPlatform }}-$(System.JobAttempt)' @@ -195,6 +202,16 @@ steps: continueOnError: true condition: succeededOrFailed() +- bash: | + if [ "$JOB_STATUS_BEFORE_VSDROPS" != "$AGENT_JOBSTATUS" ]; then + echo "VSDrops publish changed job status from '$JOB_STATUS_BEFORE_VSDROPS' to '$AGENT_JOBSTATUS'" + echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed" + fi + name: setVSDropsPublishResult + displayName: 'Set VSDrops publish result' + continueOnError: true + condition: succeededOrFailed() + - bash: | set -ex find . -name 'vsts-*.xml' || true From a80d2299356ce77e55f29374262583d191d6c4e2 Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 30 Apr 2026 12:31:03 +0200 Subject: [PATCH 2/3] [devops] Address PR review comments for VSDrops publish failure detection. - Use condition: failed('publishVSDrops') instead of comparing $AGENT_JOBSTATUS before/after, which reliably detects publish failures even when the job was already Failed. - Add Sort-Object | Select-Object -Last 1 to VSDROPS_PUBLISHED key lookup to handle retries/attempts deterministically. - Change text from '(Publish failed)' to '(:warning: Html Report Publish failed :warning:)'. - Add Pester test for VSDrops publish failed behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../automation/scripts/TestResults.Tests.ps1 | 72 +++++++++++++++++++ .../automation/scripts/TestResults.psm1 | 6 +- .../devops/automation/templates/mac/build.yml | 14 +--- .../automation/templates/tests/run-tests.yml | 14 +--- 4 files changed, 81 insertions(+), 25 deletions(-) diff --git a/tools/devops/automation/scripts/TestResults.Tests.ps1 b/tools/devops/automation/scripts/TestResults.Tests.ps1 index 1a27744780c7..0d29999bc54f 100644 --- a/tools/devops/automation/scripts/TestResults.Tests.ps1 +++ b/tools/devops/automation/scripts/TestResults.Tests.ps1 @@ -1352,4 +1352,76 @@ Test results reported success, but the tests job failed. $sonomaIdx | Should -BeLessThan $sequoiaIdx } } + + Context "VSDrops publish failed" { + It "shows publish failed text instead of VSDrops link" { + $VerbosePreference = "Continue" + $DebugPreference = "Continue" + + $vsdropsMatrix = @" +{ + "cecil": { + "LABEL": "cecil", + "TESTS_LABELS": "--label=skip-all-tests,run-cecil-tests", + "TEST_STAGE": "simulator_tests", + "LABEL_WITH_PLATFORM": "cecil", + "STATUS_CONTEXT": "VSTS: simulator tests - cecil", + "TEST_PREFIX": "simulator_testscecil", + "TEST_PLATFORM": "" + } +} +"@ + $vsdropsFailedStageDeps = @" +{ + "configure_build": { + "configure": { + "outputs": { + "test_matrix.TEST_MATRIX": "$($vsdropsMatrix.Replace("`n", "\n").Replace("`"", "\`""))" + } + } + }, + "simulator_tests": { + "tests": { + "outputs": { + "cecil.PowerShell15.TESTS_ATTEMPT": "1", + "cecil.PowerShell15.TESTS_BOT": "XAMMINI-013.Ventura", + "cecil.PowerShell15.TESTS_LABEL": "cecil", + "cecil.PowerShell15.TESTS_PLATFORM": "", + "cecil.PowerShell15.TESTS_TITLE": "cecil", + "cecil.runTests.TESTS_JOBSTATUS": "Succeeded", + "cecil.setVSDropsPublishResult.VSDROPS_PUBLISHED": "Failed" + }, + "identifier": null, + "name": "tests", + "attempt": 1, + "startTime": null, + "finishTime": null, + "state": "NotStarted", + "result": "Succeeded" + } + } +} +"@ + $testDirectory = Join-Path "." "subdir" + New-Item -Path "$testDirectory" -ItemType "directory" -Force + New-Item -Path "$testDirectory/TestSummary-simulator_testscecil-1" -Name "TestSummary.md" -Value "# :tada: All 1 tests passed :tada:" -Force + + $parallelResults = New-ParallelTestsResults -Path "$testDirectory" -StageDependencies "$vsdropsFailedStageDeps" -Context "context" -VSDropsIndex "vsdropsIndex" + + $parallelResults.IsSuccess() | Should -Be $true + + $sb = [System.Text.StringBuilder]::new() + $parallelResults.WriteComment($sb) + + Remove-Item -Path $testDirectory -Recurse + + $content = $sb.ToString() + + Write-Host $content + + $content | Should -Not -BeLike "*[Html Report (VSDrops)]*" + $content | Should -BeLike "*(:warning: Html Report Publish failed :warning:)*" + $content | Should -BeLike "*[Download]*" + } + } } diff --git a/tools/devops/automation/scripts/TestResults.psm1 b/tools/devops/automation/scripts/TestResults.psm1 index aab88e9952f3..da6b71d7107e 100644 --- a/tools/devops/automation/scripts/TestResults.psm1 +++ b/tools/devops/automation/scripts/TestResults.psm1 @@ -342,7 +342,7 @@ class ParallelTestsResults { [string] GetDownloadLinks($testResult) { $artifactUrl = "$Env:SYSTEM_TEAMFOUNDATIONCOLLECTIONURI$Env:SYSTEM_TEAMPROJECT/_apis/build/builds/$Env:BUILD_BUILDID/artifacts?artifactName=HtmlReport-$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)&api-version=6.0&`$format=zip" if ($testResult.VSDropsPublishFailed) { - $downloadInfo = "(Publish failed) [Download]($artifactUrl)" + $downloadInfo = "(:warning: Html Report Publish failed :warning:) [Download]($artifactUrl)" } else { $dropsIndex = "$($this.VSDropsIndex)/$($testResult.TestStage)$($testResult.Title)-$($testResult.Attempt)/;/tests/vsdrops_index.html" $downloadInfo = "[Html Report (VSDrops)]($dropsIndex) [Download]($artifactUrl)" @@ -594,7 +594,7 @@ class ParallelTestsResults { $platformKey = $outputs.Keys | Where-Object { $_.EndsWith(".TESTS_PLATFORM") } $attemptKey = $outputs.Keys | Where-Object { $_.EndsWith(".TESTS_ATTEMPT") } $titleKey = $outputs.Keys | Where-Object { $_.EndsWith(".TESTS_TITLE") } - $vsdropsPublishedKey = $outputs.Keys | Where-Object { $_.EndsWith(".VSDROPS_PUBLISHED") } + $vsdropsPublishedKey = $outputs.Keys | Where-Object { $_.EndsWith(".VSDROPS_PUBLISHED") } | Sort-Object | Select-Object -Last 1 } else { # matrix job $jobName = $name.Substring(0, $name.IndexOf('.')) @@ -603,7 +603,7 @@ class ParallelTestsResults { $platformKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".TESTS_PLATFORM") } $attemptKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".TESTS_ATTEMPT") } $titleKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".TESTS_TITLE") } - $vsdropsPublishedKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".VSDROPS_PUBLISHED") } + $vsdropsPublishedKey = $outputs.Keys | Where-Object { $_.StartsWith($jobName + ".") -and $_.EndsWith(".VSDROPS_PUBLISHED") } | Sort-Object | Select-Object -Last 1 } Write-Host "Keys for Label='$label' and JobName='$jobName' (dotCount=$dotCount): TitleKey='$titleKey' StatusKey=$statusKey BotKey=$botKey PlatformKey=$platformKey AttemptKey=$attemptKey" diff --git a/tools/devops/automation/templates/mac/build.yml b/tools/devops/automation/templates/mac/build.yml index f1db621a6e39..07285cb29504 100644 --- a/tools/devops/automation/templates/mac/build.yml +++ b/tools/devops/automation/templates/mac/build.yml @@ -210,12 +210,6 @@ steps: condition: succeededOrFailed() # Upload to VSDrops so that the Html Report link in the GitHub comment works. -- bash: | - echo "##vso[task.setvariable variable=JOB_STATUS_BEFORE_VSDROPS]$AGENT_JOBSTATUS" - displayName: 'Capture job status before VSDrops publish' - condition: succeededOrFailed() - continueOnError: true - - task: artifactDropTask@1 displayName: 'Publish to Artifact Services Drop' name: publishVSDrops @@ -231,14 +225,12 @@ steps: condition: succeededOrFailed() - bash: | - if [ "$JOB_STATUS_BEFORE_VSDROPS" != "$AGENT_JOBSTATUS" ]; then - echo "VSDrops publish changed job status from '$JOB_STATUS_BEFORE_VSDROPS' to '$AGENT_JOBSTATUS'" - echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed" - fi + echo "VSDrops publish step failed, setting VSDROPS_PUBLISHED=Failed" + echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed" name: setVSDropsPublishResult displayName: 'Set VSDrops publish result' continueOnError: true - condition: succeededOrFailed() + condition: failed('publishVSDrops') # Archive files for the Html Report so that the report can be easily uploaded as artifacts of the build. - task: ArchiveFiles@1 diff --git a/tools/devops/automation/templates/tests/run-tests.yml b/tools/devops/automation/templates/tests/run-tests.yml index a7b123644b79..4713e1cb5bff 100644 --- a/tools/devops/automation/templates/tests/run-tests.yml +++ b/tools/devops/automation/templates/tests/run-tests.yml @@ -182,12 +182,6 @@ steps: continueOnError: true condition: succeededOrFailed() -- bash: | - echo "##vso[task.setvariable variable=JOB_STATUS_BEFORE_VSDROPS]$AGENT_JOBSTATUS" - displayName: 'Capture job status before VSDrops publish' - condition: succeededOrFailed() - continueOnError: true - - task: artifactDropTask@1 displayName: 'Publish to Artifact Services Drop' name: publishVSDrops @@ -203,14 +197,12 @@ steps: condition: succeededOrFailed() - bash: | - if [ "$JOB_STATUS_BEFORE_VSDROPS" != "$AGENT_JOBSTATUS" ]; then - echo "VSDrops publish changed job status from '$JOB_STATUS_BEFORE_VSDROPS' to '$AGENT_JOBSTATUS'" - echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed" - fi + echo "VSDrops publish step failed, setting VSDROPS_PUBLISHED=Failed" + echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed" name: setVSDropsPublishResult displayName: 'Set VSDrops publish result' continueOnError: true - condition: succeededOrFailed() + condition: failed('publishVSDrops') - bash: | set -ex From 26352a6b684fc03a9ce01511081078e2a0503d9d Mon Sep 17 00:00:00 2001 From: Rolf Bjarne Kvinge Date: Thu, 30 Apr 2026 17:32:11 +0200 Subject: [PATCH 3/3] [devops] Fix invalid failed('publishVSDrops') condition syntax. Azure DevOps step conditions don't support failed('stepName') - that syntax only works for jobs and stages. Revert to the before/after AGENT_JOBSTATUS comparison, which is the only reliable way to detect individual step failures in Azure DevOps. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- tools/devops/automation/templates/mac/build.yml | 17 ++++++++++++++--- .../automation/templates/tests/run-tests.yml | 17 ++++++++++++++--- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/tools/devops/automation/templates/mac/build.yml b/tools/devops/automation/templates/mac/build.yml index 07285cb29504..fbc9f808b182 100644 --- a/tools/devops/automation/templates/mac/build.yml +++ b/tools/devops/automation/templates/mac/build.yml @@ -210,6 +210,12 @@ steps: condition: succeededOrFailed() # Upload to VSDrops so that the Html Report link in the GitHub comment works. +- bash: | + echo "##vso[task.setvariable variable=JOB_STATUS_BEFORE_VSDROPS]$AGENT_JOBSTATUS" + displayName: 'Capture job status before VSDrops publish' + condition: succeededOrFailed() + continueOnError: true + - task: artifactDropTask@1 displayName: 'Publish to Artifact Services Drop' name: publishVSDrops @@ -224,13 +230,18 @@ steps: continueOnError: true condition: succeededOrFailed() +# Azure DevOps doesn't support checking individual step results in step conditions +# (failed('stepName') only works for jobs/stages). Comparing AGENT_JOBSTATUS before +# and after the publish step is the best available approach to detect publish failures. - bash: | - echo "VSDrops publish step failed, setting VSDROPS_PUBLISHED=Failed" - echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed" + if [ "$JOB_STATUS_BEFORE_VSDROPS" != "$AGENT_JOBSTATUS" ]; then + echo "VSDrops publish changed job status from '$JOB_STATUS_BEFORE_VSDROPS' to '$AGENT_JOBSTATUS'" + echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed" + fi name: setVSDropsPublishResult displayName: 'Set VSDrops publish result' continueOnError: true - condition: failed('publishVSDrops') + condition: succeededOrFailed() # Archive files for the Html Report so that the report can be easily uploaded as artifacts of the build. - task: ArchiveFiles@1 diff --git a/tools/devops/automation/templates/tests/run-tests.yml b/tools/devops/automation/templates/tests/run-tests.yml index 4713e1cb5bff..b02ee6e707f8 100644 --- a/tools/devops/automation/templates/tests/run-tests.yml +++ b/tools/devops/automation/templates/tests/run-tests.yml @@ -182,6 +182,12 @@ steps: continueOnError: true condition: succeededOrFailed() +- bash: | + echo "##vso[task.setvariable variable=JOB_STATUS_BEFORE_VSDROPS]$AGENT_JOBSTATUS" + displayName: 'Capture job status before VSDrops publish' + condition: succeededOrFailed() + continueOnError: true + - task: artifactDropTask@1 displayName: 'Publish to Artifact Services Drop' name: publishVSDrops @@ -196,13 +202,18 @@ steps: continueOnError: true condition: succeededOrFailed() +# Azure DevOps doesn't support checking individual step results in step conditions +# (failed('stepName') only works for jobs/stages). Comparing AGENT_JOBSTATUS before +# and after the publish step is the best available approach to detect publish failures. - bash: | - echo "VSDrops publish step failed, setting VSDROPS_PUBLISHED=Failed" - echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed" + if [ "$JOB_STATUS_BEFORE_VSDROPS" != "$AGENT_JOBSTATUS" ]; then + echo "VSDrops publish changed job status from '$JOB_STATUS_BEFORE_VSDROPS' to '$AGENT_JOBSTATUS'" + echo "##vso[task.setvariable variable=VSDROPS_PUBLISHED;isOutput=true]Failed" + fi name: setVSDropsPublishResult displayName: 'Set VSDrops publish result' continueOnError: true - condition: failed('publishVSDrops') + condition: succeededOrFailed() - bash: | set -ex