From d8ec301215f6fabd79d9ff430ac1aa6985298e65 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Mon, 13 Jul 2020 17:09:44 -0700 Subject: [PATCH 1/2] add retry logic to log downloader --- eng/release/scripts/GetPublishUrls.ps1 | 31 ++++++++++++++++++-------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/eng/release/scripts/GetPublishUrls.ps1 b/eng/release/scripts/GetPublishUrls.ps1 index 758c20ea515..d09384029a6 100644 --- a/eng/release/scripts/GetPublishUrls.ps1 +++ b/eng/release/scripts/GetPublishUrls.ps1 @@ -8,6 +8,26 @@ param ( Set-StrictMode -version 2.0 $ErrorActionPreference = "Stop" +function Invoke-WebRequestWithAccessToken([string] $uri, [string] $accessToken, [int] $retryCount = 5) { + Write-Host "Fetching content from $uri" + $base64 = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$accessToken")) + $headers = @{ + Authorization = "Basic $base64" + } + + for ($i = 0; $i -lt $retryCount; $i++) { + try { + return Invoke-WebRequest -Method Get -Uri $uri -Headers $headers -UseBasicParsing + } + catch { + Write-Host "Invoke-WebRequest failed: $_" + Start-Sleep -Seconds 1 + } + } + + throw "Unable to fetch $uri after $retryCount tries." +} + try { # build map of all *.vsman files to their `info.buildVersion` values $manifestVersionMap = @{} @@ -21,17 +41,10 @@ try { # find all publish URLs $manifests = @() $seenManifests = @{} - $url = "https://dev.azure.com/dnceng/internal/_apis/build/builds/$buildId/logs?api-version=5.1" - $base64 = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$accessToken")) - $headers = @{ - Authorization = "Basic $base64" - } - Write-Host "Fetching log from $url" - $json = Invoke-WebRequest -Method Get -Uri $url -Headers $headers -UseBasicParsing | ConvertFrom-Json + $json = Invoke-WebRequestWithAccessToken -uri "https://dev.azure.com/dnceng/internal/_apis/build/builds/$buildId/logs?api-version=5.1" -accessToken $accessToken | ConvertFrom-Json foreach ($l in $json.value) { $logUrl = $l.url - Write-Host "Fetching log from $logUrl" - $log = (Invoke-WebRequest -Method Get -Uri $logUrl -Headers $headers -UseBasicParsing).Content + $log = (Invoke-WebRequestWithAccessToken -uri $logUrl -accessToken $accessToken).Content If ($log -Match "(https://vsdrop\.corp\.microsoft\.com/[^\r\n;]+);([^\r\n]+)\r?\n") { $manifestShortUrl = $Matches[1] $manifestName = $Matches[2] From b0db131ef941cd1d9f717eb26d72554363c5cf50 Mon Sep 17 00:00:00 2001 From: "Brett V. Forsgren" Date: Tue, 14 Jul 2020 11:41:16 -0700 Subject: [PATCH 2/2] download logs as zip --- eng/release/scripts/GetPublishUrls.ps1 | 77 ++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 12 deletions(-) diff --git a/eng/release/scripts/GetPublishUrls.ps1 b/eng/release/scripts/GetPublishUrls.ps1 index d09384029a6..897ef398f52 100644 --- a/eng/release/scripts/GetPublishUrls.ps1 +++ b/eng/release/scripts/GetPublishUrls.ps1 @@ -8,6 +8,8 @@ param ( Set-StrictMode -version 2.0 $ErrorActionPreference = "Stop" +$dropUrlRegex = "(https://vsdrop\.corp\.microsoft\.com/[^\r\n;]+);([^\r\n]+)\r?\n" + function Invoke-WebRequestWithAccessToken([string] $uri, [string] $accessToken, [int] $retryCount = 5) { Write-Host "Fetching content from $uri" $base64 = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$accessToken")) @@ -28,24 +30,15 @@ function Invoke-WebRequestWithAccessToken([string] $uri, [string] $accessToken, throw "Unable to fetch $uri after $retryCount tries." } -try { - # build map of all *.vsman files to their `info.buildVersion` values - $manifestVersionMap = @{} - Get-ChildItem -Path "$insertionDir\*" -Filter "*.vsman" | ForEach-Object { - $manifestName = Split-Path $_ -Leaf - $vsmanContents = Get-Content $_ | ConvertFrom-Json - $buildVersion = $vsmanContents.info.buildVersion - $manifestVersionMap.Add($manifestName, $buildVersion) - } - - # find all publish URLs +# this function has to download ~500 individual logs and check each one; prone to timeouts +function Get-ManifestsViaIndividualLogs([PSObject] $manifestVersionMap, [string] $buildId, [string] $accessToken) { $manifests = @() $seenManifests = @{} $json = Invoke-WebRequestWithAccessToken -uri "https://dev.azure.com/dnceng/internal/_apis/build/builds/$buildId/logs?api-version=5.1" -accessToken $accessToken | ConvertFrom-Json foreach ($l in $json.value) { $logUrl = $l.url $log = (Invoke-WebRequestWithAccessToken -uri $logUrl -accessToken $accessToken).Content - If ($log -Match "(https://vsdrop\.corp\.microsoft\.com/[^\r\n;]+);([^\r\n]+)\r?\n") { + If ($log -Match $dropUrlRegex) { $manifestShortUrl = $Matches[1] $manifestName = $Matches[2] $manifestUrl = "$manifestShortUrl;$manifestName" @@ -58,6 +51,66 @@ try { } } + return $manifests +} + +# this function only has to download 1 file and look at a very specific file +function Get-ManifestsViaZipLog([PSObject] $manifestVersionMap, [string] $buildId, [string] $accessToken) { + # create temporary location + $guid = [System.Guid]::NewGuid().ToString() + $tempDir = Join-Path ([System.IO.Path]::GetTempPath()) $guid + New-Item -ItemType Directory -Path $tempDir | Out-Null + + # download the logs + $base64 = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$accessToken")) + $headers = @{ + Authorization = "Basic $base64" + } + $uri = "https://dev.azure.com/dnceng/internal/_apis/build/builds/$buildId/logs?`$format=zip" + Invoke-WebRequest -Uri $uri -Method Get -Headers $headers -UseBasicParsing -OutFile "$tempDir/logs.zip" + + # expand the logs + New-Item -ItemType Directory -Path "$tempDir/logs" | Out-Null + Expand-Archive -Path "$tempDir/logs.zip" -DestinationPath "$tempDir/logs" + + # parse specific logs + $logDir = "$tempDir/logs" + $manifests = @() + $seenManifests = @{} + Get-ChildItem $logDir -r -inc "*Upload VSTS Drop*" | ForEach-Object { + $result = Select-String -Path $_ -Pattern "(https://vsdrop\.corp\.microsoft\.com[^;]+);(.*)" -AllMatches + $result.Matches | ForEach-Object { + $manifestShortUrl = $_.Groups[1].Value + $manifestName = $_.Groups[2].Value + $manifestUrl = "$manifestShortUrl;$manifestName" + If (-Not $seenManifests.Contains($manifestUrl)) { + $seenManifests.Add($manifestUrl, $true) + $buildVersion = $manifestVersionMap[$manifestName] + $manifestEntry = "$manifestName{$buildVersion}=$manifestUrl" + $manifests += $manifestEntry + } + } + } + + Remove-Item -Path $tempDir -Recurse + + return $manifests +} + +try { + # build map of all *.vsman files to their `info.buildVersion` values + $manifestVersionMap = @{} + Get-ChildItem -Path "$insertionDir\*" -Filter "*.vsman" | ForEach-Object { + $manifestName = Split-Path $_ -Leaf + $vsmanContents = Get-Content $_ | ConvertFrom-Json + $buildVersion = $vsmanContents.info.buildVersion + $manifestVersionMap.Add($manifestName, $buildVersion) + } + + # find all publish URLs + #$manifests = Get-ManifestsViaIndividualLogs -manifestVersionMap $manifestVersionMap -buildId $buildId -accessToken $accessToken + $manifests = Get-ManifestsViaZipLog -manifestVersionMap $manifestVersionMap -buildId $buildId -accessToken $accessToken + $final = $manifests -Join "," Write-Host "Setting InsertJsonValues to $final" Write-Host "##vso[task.setvariable variable=InsertJsonValues]$final"