From ae7678f2175c39eb75bfb5af463ca2ec582c96ca Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Thu, 3 Aug 2017 11:57:08 -0500 Subject: [PATCH 01/11] Get latest version info in build leg. --- .../security/DotNet-CLI-Security-Windows.json | 30 +++++++++++++-- .../security/Get-LatestVersion.ps1 | 37 ++++++------------- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index 35bdd379e2..881f547d65 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -237,6 +237,28 @@ "failOnStandardError": "true" } }, + { + "environment": {}, + "enabled": true, + "continueOnError": false, + "alwaysRun": false, + "displayName": "Get latest version info", + "timeoutInMinutes": 0, + "condition": "succeeded()", + "refName": "PowerShell23", + "task": { + "id": "e213ff0f-5d5c-4791-802d-52ea3e7be1f1", + "versionSpec": "1.*", + "definitionType": "task" + }, + "inputs": { + "scriptType": "filePath", + "scriptName": "$(Build.SourcesDirectory)\\build\\buildpipeline\\security\\Get-LatestVersion.ps1", + "arguments": "-Branch \"$(CodeBase)\"", + "workingFolder": "$(Build.SourcesDirectory)\\$(PB_Repo)\\build\\buildpipeline\\security\\", + "failOnStandardError": "true" + } + }, { "enabled": true, "continueOnError": true, @@ -286,8 +308,8 @@ "softwareFolder": "$(Build.SourcesDirectory)\\security", "mpdFolder": "", "softwareName": "CLI", - "softwareVersionNum": "$(CliLatestPackageId)", - "softwareBuildNum": "$(CliLatestPackageId)", + "softwareVersionNum": "$(CliPackageId)", + "softwareBuildNum": "$(CliPackageId)", "modeType": "prerelease", "noCopySymbols": "false", "noCopyBinaries": "false", @@ -318,7 +340,7 @@ "inputs": { "scriptType": "inlineScript", "scriptName": "", - "arguments": "-sha \"$(CliLatestCommitSha)\" -git \"$(PB_Git)\"", + "arguments": "-sha \"$(CliCommitSha)\" -git \"$(PB_Git)\"", "workingFolder": "$(Build.SourcesDirectory)", "inlineScript": "param($sha, $git)\n\nStart-Process \"$git\" -ArgumentList \"clean -df\" -Wait -Verbose -ErrorAction Stop\nStart-Process \"$git\" -ArgumentList \"checkout $sha\" -Wait -Verbose -ErrorAction Stop\nWrite-Host \"Checked out at $sha\"\n", "failOnStandardError": "true" @@ -678,7 +700,7 @@ "type": "TfsGit", "name": "DotNet-Cli-Trusted", "url": "https://devdiv.visualstudio.com/DevDiv/_git/DotNet-Cli-Trusted", - "defaultBranch": "refs/heads/master", + "defaultBranch": "refs/heads/sec_ext", "clean": "true", "checkoutSubmodules": false }, diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 index 5e20756f8a..b3e9060a0b 100644 --- a/build/buildpipeline/security/Get-LatestVersion.ps1 +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -2,7 +2,7 @@ .SYNOPSIS Retrieves the latest commit SHA and the corresponding package Id for the specified branch of CLI. This retrieval is achieved by downloading the latest.version file, which contains the commit SHA and package Id info. - If retrieval succeeds, then the commit is set as $env:CliLatestCommitSha, and package Id is set as $env:CliLatestPackageId. + If retrieval succeeds, then the commit is set as a VSTS Task Variable named $CliCommitSha, and similarly package Id is set as $CliPackageId. .PARAMETER $Branch Name of the CLI branch. .PARAMETER $Filename @@ -20,12 +20,6 @@ param( [string]$UrlPrefix="https://dotnetcli.blob.core.windows.net/dotnet/Sdk" ) -$latestVersionUrl = "$UrlPrefix/$Branch/$Filename" -$latestVersionFilePath = ".\latest.version" -$env:CliLatestCommitSha = "" -$env:CliLatestPackageId = "" - - function Get-VersionInfo { Write-Host "Attempting to retrieve latest version info from $latestVersionUrl" @@ -39,18 +33,7 @@ function Get-VersionInfo try { - if(Test-Path "$latestVersionFilePath") - { - Remove-Item "$latestVersionFilePath" -Force - } - - Invoke-WebRequest -Uri "$latestVersionUrl" -OutFile "$latestVersionFilePath" - - $latestVersionContent = Get-Content "$latestVersionFilePath" - $env:CliLatestCommitSha = $latestVersionContent[0] - $env:CliLatestPackageId = $latestVersionContent[1] - - break + return (Invoke-WebRequest -Uri "$latestVersionUrl").Content.Split([Environment]::NewLine, [System.StringSplitOptions]::RemoveEmptyEntries) } catch { @@ -67,15 +50,19 @@ function Get-VersionInfo } } -Get-VersionInfo +$latestVersionUrl = "$UrlPrefix/$Branch/$Filename" +$latestVersionContent = Get-VersionInfo -if (-not [string]::IsNullOrWhiteSpace($env:CliLatestCommitSha) -and -not [string]::IsNullOrWhiteSpace($env:CliLatestPackageId)) +if (-not [string]::IsNullOrWhiteSpace($latestVersionContent) -and $latestVersionContent.Length -eq 2) { - Write-Host "##vso[task.setvariable variable=CliLatestCommitSha;]$env:CliLatestCommitSha" - Write-Host "##vso[task.setvariable variable=CliLatestPackageId;]$env:CliLatestPackageId" + $CliCommitSha = $latestVersionContent[0] + $CliPackageId = $latestVersionContent[1] + + Write-Host "##vso[task.setvariable variable=$CliCommitSha;]$CliCommitSha" + Write-Host "##vso[task.setvariable variable=$CliPackageId;]$CliPackageId" - Write-Host "The latest commit SHA in CLI $Branch is $env:CliLatestCommitSha" - Write-Host "The latest package Id in CLI $Branch is $env:CliLatestPackageId" + Write-Host "The latest commit SHA in CLI $Branch is $CliCommitSha" + Write-Host "The latest package Id in CLI $Branch is $CliPackageId" } else { From 029027ba8120be4764274324f45a30feabdabb2d Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Thu, 3 Aug 2017 12:42:50 -0500 Subject: [PATCH 02/11] Specify UseBasicParsing parameter in Invoke-WebRequest. --- build/buildpipeline/security/Get-LatestVersion.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 index b3e9060a0b..b2f1a401e4 100644 --- a/build/buildpipeline/security/Get-LatestVersion.ps1 +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -33,7 +33,7 @@ function Get-VersionInfo try { - return (Invoke-WebRequest -Uri "$latestVersionUrl").Content.Split([Environment]::NewLine, [System.StringSplitOptions]::RemoveEmptyEntries) + return (Invoke-WebRequest -Uri "$latestVersionUrl" -UseBasicParsing).Content.Split([Environment]::NewLine, [System.StringSplitOptions]::RemoveEmptyEntries) } catch { From 292d348bcd38fbaf4b3335b5481b54960de3027f Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Thu, 3 Aug 2017 13:54:58 -0500 Subject: [PATCH 03/11] Display environment variables. --- build/buildpipeline/security/Get-LatestVersion.ps1 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 index b2f1a401e4..d76d679ed2 100644 --- a/build/buildpipeline/security/Get-LatestVersion.ps1 +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -63,6 +63,10 @@ if (-not [string]::IsNullOrWhiteSpace($latestVersionContent) -and $latestVersion Write-Host "The latest commit SHA in CLI $Branch is $CliCommitSha" Write-Host "The latest package Id in CLI $Branch is $CliPackageId" + + # TODO: Remove before PR. + Write-Host "List of all environment variables." + gci env: | sort name } else { From 9aa6d7575295b0ae12a2c0e4954dc8212996d7a7 Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Thu, 3 Aug 2017 16:14:03 -0500 Subject: [PATCH 04/11] Remove $ before variable name. --- build/buildpipeline/security/Get-LatestVersion.ps1 | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 index d76d679ed2..eebb57d1cb 100644 --- a/build/buildpipeline/security/Get-LatestVersion.ps1 +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -58,15 +58,11 @@ if (-not [string]::IsNullOrWhiteSpace($latestVersionContent) -and $latestVersion $CliCommitSha = $latestVersionContent[0] $CliPackageId = $latestVersionContent[1] - Write-Host "##vso[task.setvariable variable=$CliCommitSha;]$CliCommitSha" - Write-Host "##vso[task.setvariable variable=$CliPackageId;]$CliPackageId" + Write-Host "##vso[task.setvariable variable=CliCommitSha;]$CliCommitSha" + Write-Host "##vso[task.setvariable variable=CliPackageId;]$CliPackageId" Write-Host "The latest commit SHA in CLI $Branch is $CliCommitSha" Write-Host "The latest package Id in CLI $Branch is $CliPackageId" - - # TODO: Remove before PR. - Write-Host "List of all environment variables." - gci env: | sort name } else { From 46cc9c5a18e31140e17c29ac22d291a35c3489ac Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Thu, 3 Aug 2017 17:11:09 -0500 Subject: [PATCH 05/11] Rename variables. --- build/buildpipeline/security/Get-LatestVersion.ps1 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 index eebb57d1cb..a9d3fdf405 100644 --- a/build/buildpipeline/security/Get-LatestVersion.ps1 +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -2,7 +2,7 @@ .SYNOPSIS Retrieves the latest commit SHA and the corresponding package Id for the specified branch of CLI. This retrieval is achieved by downloading the latest.version file, which contains the commit SHA and package Id info. - If retrieval succeeds, then the commit is set as a VSTS Task Variable named $CliCommitSha, and similarly package Id is set as $CliPackageId. + If retrieval succeeds, then the commit is set as a VSTS Task Variable named CliLatestCommitSha, and similarly package Id is set as CliLatestPackageId. .PARAMETER $Branch Name of the CLI branch. .PARAMETER $Filename @@ -55,14 +55,14 @@ $latestVersionContent = Get-VersionInfo if (-not [string]::IsNullOrWhiteSpace($latestVersionContent) -and $latestVersionContent.Length -eq 2) { - $CliCommitSha = $latestVersionContent[0] - $CliPackageId = $latestVersionContent[1] + $CliLatestCommitSha = $latestVersionContent[0] + $CliLatestPackageId = $latestVersionContent[1] - Write-Host "##vso[task.setvariable variable=CliCommitSha;]$CliCommitSha" - Write-Host "##vso[task.setvariable variable=CliPackageId;]$CliPackageId" + Write-Host "##vso[task.setvariable variable=CliLatestCommitSha;]$CliLatestCommitSha" + Write-Host "##vso[task.setvariable variable=CliLatestPackageId;]$CliLatestPackageId" - Write-Host "The latest commit SHA in CLI $Branch is $CliCommitSha" - Write-Host "The latest package Id in CLI $Branch is $CliPackageId" + Write-Host "The latest commit SHA in CLI $Branch is $CliLatestCommitSha" + Write-Host "The latest package Id in CLI $Branch is $CliLatestPackageId" } else { From e352fbd359a499447855c92b9df1795e51381018 Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Thu, 3 Aug 2017 17:12:21 -0500 Subject: [PATCH 06/11] Switch to master branch. --- build/buildpipeline/security/DotNet-CLI-Security-Windows.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index 881f547d65..6f3c255257 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -700,7 +700,7 @@ "type": "TfsGit", "name": "DotNet-Cli-Trusted", "url": "https://devdiv.visualstudio.com/DevDiv/_git/DotNet-Cli-Trusted", - "defaultBranch": "refs/heads/sec_ext", + "defaultBranch": "refs/heads/master", "clean": "true", "checkoutSubmodules": false }, From 204a62e46ab2c690a09503e94e70a50b534ff62d Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Thu, 3 Aug 2017 17:13:35 -0500 Subject: [PATCH 07/11] Consistent variable names. --- .../buildpipeline/security/DotNet-CLI-Security-Windows.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index 6f3c255257..1226921bbf 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -308,8 +308,8 @@ "softwareFolder": "$(Build.SourcesDirectory)\\security", "mpdFolder": "", "softwareName": "CLI", - "softwareVersionNum": "$(CliPackageId)", - "softwareBuildNum": "$(CliPackageId)", + "softwareVersionNum": "$(CliLatestPackageId)", + "softwareBuildNum": "$(CliLatestPackageId)", "modeType": "prerelease", "noCopySymbols": "false", "noCopyBinaries": "false", @@ -340,7 +340,7 @@ "inputs": { "scriptType": "inlineScript", "scriptName": "", - "arguments": "-sha \"$(CliCommitSha)\" -git \"$(PB_Git)\"", + "arguments": "-sha \"$(CliLatestCommitSha)\" -git \"$(PB_Git)\"", "workingFolder": "$(Build.SourcesDirectory)", "inlineScript": "param($sha, $git)\n\nStart-Process \"$git\" -ArgumentList \"clean -df\" -Wait -Verbose -ErrorAction Stop\nStart-Process \"$git\" -ArgumentList \"checkout $sha\" -Wait -Verbose -ErrorAction Stop\nWrite-Host \"Checked out at $sha\"\n", "failOnStandardError": "true" From 8181d7730139528fe401430a014bc1a83b3e385c Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Thu, 3 Aug 2017 18:20:13 -0500 Subject: [PATCH 08/11] Addressed PR feedback. --- build/buildpipeline/security/Get-LatestVersion.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 index a9d3fdf405..13ed3e0bd5 100644 --- a/build/buildpipeline/security/Get-LatestVersion.ps1 +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -27,13 +27,14 @@ function Get-VersionInfo $retryCount = 1 $oldEap = $ErrorActionPreference - while ($retryCount -le 3) + while ($retryCount -le $retries) { $ErrorActionPreference = "Stop" try { - return (Invoke-WebRequest -Uri "$latestVersionUrl" -UseBasicParsing).Content.Split([Environment]::NewLine, [System.StringSplitOptions]::RemoveEmptyEntries) + $content = (Invoke-WebRequest -Uri "$latestVersionUrl" -UseBasicParsing).Content + return $content.Split([Environment]::NewLine, [System.StringSplitOptions]::RemoveEmptyEntries) } catch { From bd8f1c01268813e9ead9403ee070e8187127726d Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Thu, 3 Aug 2017 18:23:03 -0500 Subject: [PATCH 09/11] Switch to test branch. --- build/buildpipeline/security/DotNet-CLI-Security-Windows.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index 1226921bbf..b1db59dd9b 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -700,7 +700,7 @@ "type": "TfsGit", "name": "DotNet-Cli-Trusted", "url": "https://devdiv.visualstudio.com/DevDiv/_git/DotNet-Cli-Trusted", - "defaultBranch": "refs/heads/master", + "defaultBranch": "refs/heads/sec_ext", "clean": "true", "checkoutSubmodules": false }, From bdfbd013f1830bc81a91d6e28b60ea69f820d416 Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Fri, 4 Aug 2017 10:26:39 -0500 Subject: [PATCH 10/11] PR feedback. --- build/buildpipeline/security/DotNet-CLI-Security-Windows.json | 2 +- build/buildpipeline/security/Get-LatestVersion.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index b1db59dd9b..dc5871e5bb 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -255,7 +255,7 @@ "scriptType": "filePath", "scriptName": "$(Build.SourcesDirectory)\\build\\buildpipeline\\security\\Get-LatestVersion.ps1", "arguments": "-Branch \"$(CodeBase)\"", - "workingFolder": "$(Build.SourcesDirectory)\\$(PB_Repo)\\build\\buildpipeline\\security\\", + "workingFolder": "", "failOnStandardError": "true" } }, diff --git a/build/buildpipeline/security/Get-LatestVersion.ps1 b/build/buildpipeline/security/Get-LatestVersion.ps1 index 13ed3e0bd5..2f921ef3a8 100644 --- a/build/buildpipeline/security/Get-LatestVersion.ps1 +++ b/build/buildpipeline/security/Get-LatestVersion.ps1 @@ -54,7 +54,7 @@ function Get-VersionInfo $latestVersionUrl = "$UrlPrefix/$Branch/$Filename" $latestVersionContent = Get-VersionInfo -if (-not [string]::IsNullOrWhiteSpace($latestVersionContent) -and $latestVersionContent.Length -eq 2) +if ($latestVersionContent -ne $null -and $latestVersionContent.Length -eq 2) { $CliLatestCommitSha = $latestVersionContent[0] $CliLatestPackageId = $latestVersionContent[1] From 4fc03af1e58b29331831ac411908e899825ebcbc Mon Sep 17 00:00:00 2001 From: Ravi Eda Date: Fri, 4 Aug 2017 10:33:47 -0500 Subject: [PATCH 11/11] Switch to master. --- build/buildpipeline/security/DotNet-CLI-Security-Windows.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json index dc5871e5bb..109ade2d2e 100644 --- a/build/buildpipeline/security/DotNet-CLI-Security-Windows.json +++ b/build/buildpipeline/security/DotNet-CLI-Security-Windows.json @@ -700,7 +700,7 @@ "type": "TfsGit", "name": "DotNet-Cli-Trusted", "url": "https://devdiv.visualstudio.com/DevDiv/_git/DotNet-Cli-Trusted", - "defaultBranch": "refs/heads/sec_ext", + "defaultBranch": "refs/heads/master", "clean": "true", "checkoutSubmodules": false },