From 314358ad7127d0e6f6abec6ff84ad499f113cf6e Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Wed, 2 Aug 2017 13:11:51 -0500 Subject: [PATCH 1/6] Pass latest version info to build legs. --- .../security/DotNet-CLI-Security-Windows.json | 10 +-- .../security/Get-LatestVersion.ps1 | 79 +++++++++++++++++++ 2 files changed, 84 insertions(+), 5 deletions(-) create mode 100644 build/buildpipeline/security/Get-LatestVersion.ps1 diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index ff52ffeb55..7a5a242c08 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -286,8 +286,8 @@ "softwareFolder": "$(Build.SourcesDirectory)\\security", "mpdFolder": "", "softwareName": "CLI", - "softwareVersionNum": "$(PB_BuildNumber)", - "softwareBuildNum": "$(PB_BuildNumber)", + "softwareVersionNum": "$(CliLatestPackageId)", + "softwareBuildNum": "$(CliLatestPackageId)", "modeType": "prerelease", "noCopySymbols": "false", "noCopyBinaries": "false", @@ -318,9 +318,9 @@ "inputs": { "scriptType": "inlineScript", "scriptName": "", - "arguments": "-SrcDir \"$(Build.SourcesDirectory)\" -git \"$(PB_Git)\"", + "arguments": "-sha\"$(CliLatestCommitSha)\" -git \"$(PB_Git)\"\"", "workingFolder": "$(Build.SourcesDirectory)", - "inlineScript": "param($SrcDir, $git)\n$secDir = Join-Path \"$SrcDir\" \"security\"\n$shaFile= Join-Path \"$secDir\" \"latest.version\"\n$sha = gc \"$shaFile\" -first 1\n\nif ([string]::IsNullOrWhiteSpace($sha))\n{ Write-Error \"Unable to determine latest commit SHA.\" }\n\nStart-Process \"$git\" -ArgumentList \"clean -df\" -Wait -Verbose -ErrorAction Stop\nStart-Process \"$git\" -ArgumentList \"checkout $sha\" -Wait -Verbose -ErrorAction Stop\nWrite-Host \"Checked out at $sha\"\n", + "inlineScript": "param($sha, $git)\n\nStart-Process \"$git\" -ArgumentList \"clean -df\" -Wait -Verbose -ErrorAction Stop\nStart-Process \"$git\" -ArgumentList \"checkout $sha\" -Wait -Verbose -ErrorAction Stop\nWrite-Host \"Checked out at $sha\"\n", "failOnStandardError": "true" } }, @@ -678,7 +678,7 @@ "type": "TfsGit", "name": "DotNet-Cli-Trusted", "url": "https://devdiv.visualstudio.com/DevDiv/_git/DotNet-Cli-Trusted", - "defaultBranch": "refs/heads/master", + "defaultBranch": "refs/heads/sec_ext", "clean": "true", "checkoutSubmodules": false }, diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 new file mode 100644 index 0000000000..4fa104db64 --- /dev/null +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -0,0 +1,79 @@ +<# +.SYNOPSIS + Retrieves the latest commit SHA and the corresponding package Id for the specified branch of CLI. + This retrieval is achieved by downloading the latest.version file, which contains the commit SHA and package Id info. + If retrieval succeeds, then the commit is set as $env:CliLatestCommitSha, and package Id is set as $env:CliLatestPackageId. +.PARAMETER $Branch + Name of the CLI branch. +.PARAMETER $Filename + Name of the file that contains latest version info i.e. commit SHA and package Id. + If not specified, then the default value is latest.version +.PARAMETER $UrlPrefix + URL prefix for $Filename. + If not specified, then the default value is https://dotnetcli.blob.core.windows.net/dotnet/Sdk +#> + +param( + [Parameter(Mandatory=$true)] + [string]$Branch, + [string]$Filename="latest.version", + [string]$UrlPrefix="https://dotnetcli.blob.core.windows.net/dotnet/Sdk" +) + +$latestVersionUrl = "$UrlPrefix/$Branch/$Filename" +$latestVersionFilePath = ".\latest.version" +$env:CliLatestCommitSha = "" +$env:CliLatestPackageId = "" + +function Get-VersionInfo +{ + $retries = 3 + $retryCount = 0 + + while ($retryCount -le 3) + { + $oldEap = $ErrorActionPreference + $ErrorActionPreference = "Stop" + + try + { + if(Test-Path "$latestVersionFilePath") + { + Remove-Item "$latestVersionFilePath" -Force + } + + Invoke-WebRequest -Uri "$latestVersionUrl" -OutFile "$latestVersionFilePath" + + $latestVersionContent = Get-Content "$latestVersionFilePath" + $env:CliLatestCommitSha = $latestVersionContent[0] + $env:CliLatestPackageId = $latestVersionContent[1] + + break + } + catch + { + Sleep -Seconds (Get-Random -minimum 3 -maximum 10) + Write-Host "Exception occurred while attempting to get latest version info from $latestVersionUrl. $_" + Write-Host "Retry $retryCount of $retries" + } + finally + { + $ErrorActionPreference = $oldEap + } + } +} + +Get-VersionInfo + +if (-not [string]::IsNullOrWhiteSpace($env:CliLatestCommitSha) -and -not [string]::IsNullOrWhiteSpace($env:CliLatestPackageId)) +{ + Write-Host "##vso[task.setvariable variable=CliLatestCommitSha;]$env:CliLatestCommitSha" + Write-Host "##vso[task.setvariable variable=CliLatestPackageId;]$env:CliLatestPackageId" + + Write-Host "The latest commit SHA in CLI $Branch is $env:CliLatestCommitSha" + Write-Host "The latest package Id in CLI $Branch is $env:CliLatestPackageId" +} +else +{ + Write-Error "Unable to get latest version info from $latestVersionUrl" +} From a5d425370c0e27aa24dc70f4b1960e80a2c9d86c Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Wed, 2 Aug 2017 13:16:53 -0500 Subject: [PATCH 2/6] Increment retry count. --- build/buildpipeline/security/Get-LatestVersion.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 index 4fa104db64..8cc1a53dfb 100644 --- a/build/buildpipeline/security/Get-LatestVersion.ps1 +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -52,6 +52,7 @@ function Get-VersionInfo } catch { + $retryCount++ Sleep -Seconds (Get-Random -minimum 3 -maximum 10) Write-Host "Exception occurred while attempting to get latest version info from $latestVersionUrl. $_" Write-Host "Retry $retryCount of $retries" From 34b9a281932dfd96fcd0757fecca8e093f370770 Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Wed, 2 Aug 2017 14:40:29 -0500 Subject: [PATCH 3/6] Remove extra quote. --- build/buildpipeline/security/DotNet-CLI-Security-Windows.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index 7a5a242c08..ab7bbae085 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -318,7 +318,7 @@ "inputs": { "scriptType": "inlineScript", "scriptName": "", - "arguments": "-sha\"$(CliLatestCommitSha)\" -git \"$(PB_Git)\"\"", + "arguments": "-sha\"$(CliLatestCommitSha)\" -git \"$(PB_Git)\"", "workingFolder": "$(Build.SourcesDirectory)", "inlineScript": "param($sha, $git)\n\nStart-Process \"$git\" -ArgumentList \"clean -df\" -Wait -Verbose -ErrorAction Stop\nStart-Process \"$git\" -ArgumentList \"checkout $sha\" -Wait -Verbose -ErrorAction Stop\nWrite-Host \"Checked out at $sha\"\n", "failOnStandardError": "true" From 0343f3c4ddcb9fcc05b47c8b7dc08c6d1c24bc67 Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Wed, 2 Aug 2017 15:17:48 -0500 Subject: [PATCH 4/6] Add missing space. --- build/buildpipeline/security/DotNet-CLI-Security-Windows.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index ab7bbae085..78d6a9b3bc 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -318,7 +318,7 @@ "inputs": { "scriptType": "inlineScript", "scriptName": "", - "arguments": "-sha\"$(CliLatestCommitSha)\" -git \"$(PB_Git)\"", + "arguments": "-sha \"$(CliLatestCommitSha)\" -git \"$(PB_Git)\"", "workingFolder": "$(Build.SourcesDirectory)", "inlineScript": "param($sha, $git)\n\nStart-Process \"$git\" -ArgumentList \"clean -df\" -Wait -Verbose -ErrorAction Stop\nStart-Process \"$git\" -ArgumentList \"checkout $sha\" -Wait -Verbose -ErrorAction Stop\nWrite-Host \"Checked out at $sha\"\n", "failOnStandardError": "true" From b4717e6c3cdb3d8d419f84dcbced945a3f22f143 Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Wed, 2 Aug 2017 15:34:39 -0500 Subject: [PATCH 5/6] Refactoring of Get-LatestVersion.ps1 --- build/buildpipeline/security/Get-LatestVersion.ps1 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 index 8cc1a53dfb..ce2b54aaf4 100644 --- a/build/buildpipeline/security/Get-LatestVersion.ps1 +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -25,14 +25,16 @@ $latestVersionFilePath = ".\latest.version" $env:CliLatestCommitSha = "" $env:CliLatestPackageId = "" + function Get-VersionInfo { + Write-Host "Attempting to retrieve latest version info from $latestVersionUrl" $retries = 3 $retryCount = 0 + $oldEap = $ErrorActionPreference while ($retryCount -le 3) { - $oldEap = $ErrorActionPreference $ErrorActionPreference = "Stop" try @@ -52,7 +54,6 @@ function Get-VersionInfo } catch { - $retryCount++ Sleep -Seconds (Get-Random -minimum 3 -maximum 10) Write-Host "Exception occurred while attempting to get latest version info from $latestVersionUrl. $_" Write-Host "Retry $retryCount of $retries" @@ -61,6 +62,8 @@ function Get-VersionInfo { $ErrorActionPreference = $oldEap } + + $retryCount++ } } From 220b71ca4d870165bbd7043694b8c0281bb73877 Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Wed, 2 Aug 2017 15:37:03 -0500 Subject: [PATCH 6/6] Retry 3 times to get the latest version info. --- build/buildpipeline/security/Get-LatestVersion.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 index ce2b54aaf4..5e20756f8a 100644 --- a/build/buildpipeline/security/Get-LatestVersion.ps1 +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -30,7 +30,7 @@ function Get-VersionInfo { Write-Host "Attempting to retrieve latest version info from $latestVersionUrl" $retries = 3 - $retryCount = 0 + $retryCount = 1 $oldEap = $ErrorActionPreference while ($retryCount -le 3)