diff --git a/eng/release/scripts/GetPublishUrls.ps1 b/eng/release/scripts/GetPublishUrls.ps1 index 758c20ea515..897ef398f52 100644 --- a/eng/release/scripts/GetPublishUrls.ps1 +++ b/eng/release/scripts/GetPublishUrls.ps1 @@ -8,31 +8,37 @@ param ( Set-StrictMode -version 2.0 $ErrorActionPreference = "Stop" -try { - # build map of all *.vsman files to their `info.buildVersion` values - $manifestVersionMap = @{} - Get-ChildItem -Path "$insertionDir\*" -Filter "*.vsman" | ForEach-Object { - $manifestName = Split-Path $_ -Leaf - $vsmanContents = Get-Content $_ | ConvertFrom-Json - $buildVersion = $vsmanContents.info.buildVersion - $manifestVersionMap.Add($manifestName, $buildVersion) - } +$dropUrlRegex = "(https://vsdrop\.corp\.microsoft\.com/[^\r\n;]+);([^\r\n]+)\r?\n" - # find all publish URLs - $manifests = @() - $seenManifests = @{} - $url = "https://dev.azure.com/dnceng/internal/_apis/build/builds/$buildId/logs?api-version=5.1" +function Invoke-WebRequestWithAccessToken([string] $uri, [string] $accessToken, [int] $retryCount = 5) { + Write-Host "Fetching content from $uri" $base64 = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$accessToken")) $headers = @{ Authorization = "Basic $base64" } - Write-Host "Fetching log from $url" - $json = Invoke-WebRequest -Method Get -Uri $url -Headers $headers -UseBasicParsing | ConvertFrom-Json + + for ($i = 0; $i -lt $retryCount; $i++) { + try { + return Invoke-WebRequest -Method Get -Uri $uri -Headers $headers -UseBasicParsing + } + catch { + Write-Host "Invoke-WebRequest failed: $_" + Start-Sleep -Seconds 1 + } + } + + throw "Unable to fetch $uri after $retryCount tries." +} + +# this function has to download ~500 individual logs and check each one; prone to timeouts +function Get-ManifestsViaIndividualLogs([PSObject] $manifestVersionMap, [string] $buildId, [string] $accessToken) { + $manifests = @() + $seenManifests = @{} + $json = Invoke-WebRequestWithAccessToken -uri "https://dev.azure.com/dnceng/internal/_apis/build/builds/$buildId/logs?api-version=5.1" -accessToken $accessToken | ConvertFrom-Json foreach ($l in $json.value) { $logUrl = $l.url - Write-Host "Fetching log from $logUrl" - $log = (Invoke-WebRequest -Method Get -Uri $logUrl -Headers $headers -UseBasicParsing).Content - If ($log -Match "(https://vsdrop\.corp\.microsoft\.com/[^\r\n;]+);([^\r\n]+)\r?\n") { + $log = (Invoke-WebRequestWithAccessToken -uri $logUrl -accessToken $accessToken).Content + If ($log -Match $dropUrlRegex) { $manifestShortUrl = $Matches[1] $manifestName = $Matches[2] $manifestUrl = "$manifestShortUrl;$manifestName" @@ -45,6 +51,66 @@ try { } } + return $manifests +} + +# this function only has to download 1 file and look at a very specific file +function Get-ManifestsViaZipLog([PSObject] $manifestVersionMap, [string] $buildId, [string] $accessToken) { + # create temporary location + $guid = [System.Guid]::NewGuid().ToString() + $tempDir = Join-Path ([System.IO.Path]::GetTempPath()) $guid + New-Item -ItemType Directory -Path $tempDir | Out-Null + + # download the logs + $base64 = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$accessToken")) + $headers = @{ + Authorization = "Basic $base64" + } + $uri = "https://dev.azure.com/dnceng/internal/_apis/build/builds/$buildId/logs?`$format=zip" + Invoke-WebRequest -Uri $uri -Method Get -Headers $headers -UseBasicParsing -OutFile "$tempDir/logs.zip" + + # expand the logs + New-Item -ItemType Directory -Path "$tempDir/logs" | Out-Null + Expand-Archive -Path "$tempDir/logs.zip" -DestinationPath "$tempDir/logs" + + # parse specific logs + $logDir = "$tempDir/logs" + $manifests = @() + $seenManifests = @{} + Get-ChildItem $logDir -r -inc "*Upload VSTS Drop*" | ForEach-Object { + $result = Select-String -Path $_ -Pattern "(https://vsdrop\.corp\.microsoft\.com[^;]+);(.*)" -AllMatches + $result.Matches | ForEach-Object { + $manifestShortUrl = $_.Groups[1].Value + $manifestName = $_.Groups[2].Value + $manifestUrl = "$manifestShortUrl;$manifestName" + If (-Not $seenManifests.Contains($manifestUrl)) { + $seenManifests.Add($manifestUrl, $true) + $buildVersion = $manifestVersionMap[$manifestName] + $manifestEntry = "$manifestName{$buildVersion}=$manifestUrl" + $manifests += $manifestEntry + } + } + } + + Remove-Item -Path $tempDir -Recurse + + return $manifests +} + +try { + # build map of all *.vsman files to their `info.buildVersion` values + $manifestVersionMap = @{} + Get-ChildItem -Path "$insertionDir\*" -Filter "*.vsman" | ForEach-Object { + $manifestName = Split-Path $_ -Leaf + $vsmanContents = Get-Content $_ | ConvertFrom-Json + $buildVersion = $vsmanContents.info.buildVersion + $manifestVersionMap.Add($manifestName, $buildVersion) + } + + # find all publish URLs + #$manifests = Get-ManifestsViaIndividualLogs -manifestVersionMap $manifestVersionMap -buildId $buildId -accessToken $accessToken + $manifests = Get-ManifestsViaZipLog -manifestVersionMap $manifestVersionMap -buildId $buildId -accessToken $accessToken + $final = $manifests -Join "," Write-Host "Setting InsertJsonValues to $final" Write-Host "##vso[task.setvariable variable=InsertJsonValues]$final"