From 19a9c174f08cec5d66769ade5dbdd9b7ad2e4cd1 Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Wed, 2 Aug 2017 16:06:11 -0500 Subject: [PATCH 1/4] Get latest version info from latest.version file. (#5) --- .../security/DotNet-CLI-Security-Windows.json | 10 +-- .../security/Get-LatestVersion.ps1 | 83 +++++++++++++++++++ 2 files changed, 88 insertions(+), 5 deletions(-) create mode 100644 build/buildpipeline/security/Get-LatestVersion.ps1 diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index ff52ffeb55..78d6a9b3bc 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -286,8 +286,8 @@ "softwareFolder": "$(Build.SourcesDirectory)\\security", "mpdFolder": "", "softwareName": "CLI", - "softwareVersionNum": "$(PB_BuildNumber)", - "softwareBuildNum": "$(PB_BuildNumber)", + "softwareVersionNum": "$(CliLatestPackageId)", + "softwareBuildNum": "$(CliLatestPackageId)", "modeType": "prerelease", "noCopySymbols": "false", "noCopyBinaries": "false", @@ -318,9 +318,9 @@ "inputs": { "scriptType": "inlineScript", "scriptName": "", - "arguments": "-SrcDir \"$(Build.SourcesDirectory)\" -git \"$(PB_Git)\"", + "arguments": "-sha \"$(CliLatestCommitSha)\" -git \"$(PB_Git)\"", "workingFolder": "$(Build.SourcesDirectory)", - "inlineScript": "param($SrcDir, $git)\n$secDir = Join-Path \"$SrcDir\" \"security\"\n$shaFile= Join-Path \"$secDir\" \"latest.version\"\n$sha = gc \"$shaFile\" -first 1\n\nif ([string]::IsNullOrWhiteSpace($sha))\n{ Write-Error \"Unable to determine latest commit SHA.\" }\n\nStart-Process \"$git\" -ArgumentList \"clean -df\" -Wait -Verbose -ErrorAction Stop\nStart-Process \"$git\" -ArgumentList \"checkout $sha\" -Wait -Verbose -ErrorAction Stop\nWrite-Host \"Checked out at $sha\"\n", + "inlineScript": "param($sha, $git)\n\nStart-Process \"$git\" -ArgumentList \"clean -df\" -Wait -Verbose -ErrorAction Stop\nStart-Process \"$git\" -ArgumentList \"checkout $sha\" -Wait -Verbose -ErrorAction Stop\nWrite-Host \"Checked out at $sha\"\n", "failOnStandardError": "true" } }, @@ -678,7 +678,7 @@ "type": "TfsGit", "name": "DotNet-Cli-Trusted", "url": "https://devdiv.visualstudio.com/DevDiv/_git/DotNet-Cli-Trusted", - "defaultBranch": "refs/heads/master", + "defaultBranch": "refs/heads/sec_ext", "clean": "true", "checkoutSubmodules": false }, diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 new file mode 100644 index 0000000000..5e20756f8a --- /dev/null +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -0,0 +1,83 @@ +<# +.SYNOPSIS + Retrieves the latest commit SHA and the corresponding package Id for the specified branch of CLI. + This retrieval is achieved by downloading the latest.version file, which contains the commit SHA and package Id info. + If retrieval succeeds, then the commit is set as $env:CliLatestCommitSha, and package Id is set as $env:CliLatestPackageId. +.PARAMETER $Branch + Name of the CLI branch. +.PARAMETER $Filename + Name of the file that contains latest version info i.e. commit SHA and package Id. + If not specified, then the default value is latest.version +.PARAMETER $UrlPrefix + URL prefix for $Filename. + If not specified, then the default value is https://dotnetcli.blob.core.windows.net/dotnet/Sdk +#> + +param( + [Parameter(Mandatory=$true)] + [string]$Branch, + [string]$Filename="latest.version", + [string]$UrlPrefix="https://dotnetcli.blob.core.windows.net/dotnet/Sdk" +) + +$latestVersionUrl = "$UrlPrefix/$Branch/$Filename" +$latestVersionFilePath = ".\latest.version" +$env:CliLatestCommitSha = "" +$env:CliLatestPackageId = "" + + +function Get-VersionInfo +{ + Write-Host "Attempting to retrieve latest version info from $latestVersionUrl" + $retries = 3 + $retryCount = 1 + $oldEap = $ErrorActionPreference + + while ($retryCount -le 3) + { + $ErrorActionPreference = "Stop" + + try + { + if(Test-Path "$latestVersionFilePath") + { + Remove-Item "$latestVersionFilePath" -Force + } + + Invoke-WebRequest -Uri "$latestVersionUrl" -OutFile "$latestVersionFilePath" + + $latestVersionContent = Get-Content "$latestVersionFilePath" + $env:CliLatestCommitSha = $latestVersionContent[0] + $env:CliLatestPackageId = $latestVersionContent[1] + + break + } + catch + { + Sleep -Seconds (Get-Random -minimum 3 -maximum 10) + Write-Host "Exception occurred while attempting to get latest version info from $latestVersionUrl. $_" + Write-Host "Retry $retryCount of $retries" + } + finally + { + $ErrorActionPreference = $oldEap + } + + $retryCount++ + } +} + +Get-VersionInfo + +if (-not [string]::IsNullOrWhiteSpace($env:CliLatestCommitSha) -and -not [string]::IsNullOrWhiteSpace($env:CliLatestPackageId)) +{ + Write-Host "##vso[task.setvariable variable=CliLatestCommitSha;]$env:CliLatestCommitSha" + Write-Host "##vso[task.setvariable variable=CliLatestPackageId;]$env:CliLatestPackageId" + + Write-Host "The latest commit SHA in CLI $Branch is $env:CliLatestCommitSha" + Write-Host "The latest package Id in CLI $Branch is $env:CliLatestPackageId" +} +else +{ + Write-Error "Unable to get latest version info from $latestVersionUrl" +} From e3bca329cf431835409057f3e238136dee4072f3 Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Thu, 3 Aug 2017 09:56:55 -0500 Subject: [PATCH 2/4] Switch to master branch. --- build/buildpipeline/security/DotNet-CLI-Security-Windows.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index 78d6a9b3bc..35bdd379e2 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -678,7 +678,7 @@ "type": "TfsGit", "name": "DotNet-Cli-Trusted", "url": "https://devdiv.visualstudio.com/DevDiv/_git/DotNet-Cli-Trusted", - "defaultBranch": "refs/heads/sec_ext", + "defaultBranch": "refs/heads/master", "clean": "true", "checkoutSubmodules": false }, From 0d837ca86fe70be832ce1a420280cc4478510abc Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Thu, 3 Aug 2017 17:16:03 -0500 Subject: [PATCH 3/4] Addressed PR feedback (#6) --- .../security/DotNet-CLI-Security-Windows.json | 22 +++++++++++ .../security/Get-LatestVersion.ps1 | 37 ++++++------------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index 35bdd379e2..1226921bbf 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -237,6 +237,28 @@ "failOnStandardError": "true" } }, + { + "environment": {}, + "enabled": true, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Get latest version info", + "timeoutInMinutes": 0, + "condition": "succeeded()", + "refName": "PowerShell23", + "task": { + "id": "e213ff0f-5d5c-4791-802d-52ea3e7be1f1", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "scriptType": "filePath", + "scriptName": "$(Build.SourcesDirectory)\\build\\buildpipeline\\security\\Get-LatestVersion.ps1", + "arguments": "-Branch \"$(CodeBase)\"", + "workingFolder": "$(Build.SourcesDirectory)\\$(PB_Repo)\\build\\buildpipeline\\security\\", + "failOnStandardError": "true" + } + }, { "enabled": true, "continueOnError": true, diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 index 5e20756f8a..a9d3fdf405 100644 --- a/build/buildpipeline/security/Get-LatestVersion.ps1 +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -2,7 +2,7 @@ .SYNOPSIS Retrieves the latest commit SHA and the corresponding package Id for the specified branch of CLI. This retrieval is achieved by downloading the latest.version file, which contains the commit SHA and package Id info. - If retrieval succeeds, then the commit is set as $env:CliLatestCommitSha, and package Id is set as $env:CliLatestPackageId. + If retrieval succeeds, then the commit is set as a VSTS Task Variable named CliLatestCommitSha, and similarly package Id is set as CliLatestPackageId. .PARAMETER $Branch Name of the CLI branch. .PARAMETER $Filename @@ -20,12 +20,6 @@ param( [string]$UrlPrefix="https://dotnetcli.blob.core.windows.net/dotnet/Sdk" ) -$latestVersionUrl = "$UrlPrefix/$Branch/$Filename" -$latestVersionFilePath = ".\latest.version" -$env:CliLatestCommitSha = "" -$env:CliLatestPackageId = "" - - function Get-VersionInfo { Write-Host "Attempting to retrieve latest version info from $latestVersionUrl" @@ -39,18 +33,7 @@ function Get-VersionInfo try { - if(Test-Path "$latestVersionFilePath") - { - Remove-Item "$latestVersionFilePath" -Force - } - - Invoke-WebRequest -Uri "$latestVersionUrl" -OutFile "$latestVersionFilePath" - - $latestVersionContent = Get-Content "$latestVersionFilePath" - $env:CliLatestCommitSha = $latestVersionContent[0] - $env:CliLatestPackageId = $latestVersionContent[1] - - break + return (Invoke-WebRequest -Uri "$latestVersionUrl" -UseBasicParsing).Content.Split([Environment]::NewLine, [System.StringSplitOptions]::RemoveEmptyEntries) } catch { @@ -67,15 +50,19 @@ function Get-VersionInfo } } -Get-VersionInfo +$latestVersionUrl = "$UrlPrefix/$Branch/$Filename" +$latestVersionContent = Get-VersionInfo -if (-not [string]::IsNullOrWhiteSpace($env:CliLatestCommitSha) -and -not [string]::IsNullOrWhiteSpace($env:CliLatestPackageId)) +if (-not [string]::IsNullOrWhiteSpace($latestVersionContent) -and $latestVersionContent.Length -eq 2) { - Write-Host "##vso[task.setvariable variable=CliLatestCommitSha;]$env:CliLatestCommitSha" - Write-Host "##vso[task.setvariable variable=CliLatestPackageId;]$env:CliLatestPackageId" + $CliLatestCommitSha = $latestVersionContent[0] + $CliLatestPackageId = $latestVersionContent[1] + + Write-Host "##vso[task.setvariable variable=CliLatestCommitSha;]$CliLatestCommitSha" + Write-Host "##vso[task.setvariable variable=CliLatestPackageId;]$CliLatestPackageId" - Write-Host "The latest commit SHA in CLI $Branch is $env:CliLatestCommitSha" - Write-Host "The latest package Id in CLI $Branch is $env:CliLatestPackageId" + Write-Host "The latest commit SHA in CLI $Branch is $CliLatestCommitSha" + Write-Host "The latest package Id in CLI $Branch is $CliLatestPackageId" } else { From d6bd18eb72c646e70ad6486cdf528211340dceda Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Fri, 4 Aug 2017 10:37:26 -0500 Subject: [PATCH 4/4] Addressed PR feedback - part 2. (#7) --- .../security/DotNet-CLI-Security-Windows.json | 2 +- build/buildpipeline/security/Get-LatestVersion.ps1 | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index 1226921bbf..109ade2d2e 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -255,7 +255,7 @@ "scriptType": "filePath", "scriptName": "$(Build.SourcesDirectory)\\build\\buildpipeline\\security\\Get-LatestVersion.ps1", "arguments": "-Branch \"$(CodeBase)\"", - "workingFolder": "$(Build.SourcesDirectory)\\$(PB_Repo)\\build\\buildpipeline\\security\\", + "workingFolder": "", "failOnStandardError": "true" } }, diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 index a9d3fdf405..2f921ef3a8 100644 --- a/build/buildpipeline/security/Get-LatestVersion.ps1 +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -27,13 +27,14 @@ function Get-VersionInfo $retryCount = 1 $oldEap = $ErrorActionPreference - while ($retryCount -le 3) + while ($retryCount -le $retries) { $ErrorActionPreference = "Stop" try { - return (Invoke-WebRequest -Uri "$latestVersionUrl" -UseBasicParsing).Content.Split([Environment]::NewLine, [System.StringSplitOptions]::RemoveEmptyEntries) + $content = (Invoke-WebRequest -Uri "$latestVersionUrl" -UseBasicParsing).Content + return $content.Split([Environment]::NewLine, [System.StringSplitOptions]::RemoveEmptyEntries) } catch { @@ -53,7 +54,7 @@ function Get-VersionInfo $latestVersionUrl = "$UrlPrefix/$Branch/$Filename" $latestVersionContent = Get-VersionInfo -if (-not [string]::IsNullOrWhiteSpace($latestVersionContent) -and $latestVersionContent.Length -eq 2) +if ($latestVersionContent -ne $null -and $latestVersionContent.Length -eq 2) { $CliLatestCommitSha = $latestVersionContent[0] $CliLatestPackageId = $latestVersionContent[1]