diff --git a/.azuredevops/pipelineTemplates/jobs.validateModuleDeployment.yml b/.azuredevops/pipelineTemplates/jobs.validateModuleDeployment.yml index 61187f1c00..753617f7ee 100644 --- a/.azuredevops/pipelineTemplates/jobs.validateModuleDeployment.yml +++ b/.azuredevops/pipelineTemplates/jobs.validateModuleDeployment.yml @@ -293,20 +293,20 @@ jobs: $res = New-TemplateDeployment @functionInput -Verbose # Get deployment name - $deploymentName = $res.deploymentName - Write-Verbose "Deployment name: $deploymentName" -Verbose - Write-Host "##vso[task.setvariable variable=deploymentName]$deploymentName" - Write-Host "##vso[task.setvariable variable=deploymentName;isOutput=true]$deploymentName" + $deploymentNames = $res.DeploymentNames | ConvertTo-Json -Compress + Write-Verbose "Deployment name(s) [$deploymentNames]" -Verbose + Write-Host "##vso[task.setvariable variable=deploymentNames]$deploymentNames" + Write-Host "##vso[task.setvariable variable=deploymentNames;isOutput=true]$deploymentNames" # Populate further outputs - $deploymentOutputHash=@{} + $deploymentOutputHashTable=@{} foreach ($outputKey in $res.deploymentOutput.Keys) { Write-Output ('##vso[task.setvariable variable={0}]{1}' -f $outputKey, $res.deploymentOutput[$outputKey].Value) - $deploymentOutputHash.add($outputKey,$res.deploymentOutput[$outputKey].Value) + $deploymentOutputHashTable.add($outputKey,$res.deploymentOutput[$outputKey].Value) } - $deploymentOutput = $deploymentOutputHash | ConvertTo-Json -Compress -Depth 100 + $deploymentOutput = $deploymentOutputHashTable | ConvertTo-Json -Compress -Depth 100 Write-Verbose "Deployment output: $deploymentOutput" -Verbose if ($res.ContainsKey('exception')) { @@ -318,7 +318,7 @@ jobs: #------------------ - task: AzurePowerShell@5 displayName: 'Remove deployed resources via [${{ parameters.serviceConnection }}]' - condition: and(succeededOrFailed(), eq('${{ parameters.removeDeployment }}', 'True'), not(eq(variables['deploymentName'],'')), not(startsWith(variables['deploymentName'], 'variables[' ))) + condition: and(succeededOrFailed(), eq('${{ parameters.removeDeployment }}', 'True'), not(eq(variables['deploymentNames'],'')), not(startsWith(variables['deploymentNames'], 'variables[' ))) inputs: azureSubscription: ${{ parameters.serviceConnection }} azurePowerShellVersion: ${{ parameters.azurePowerShellVersion }} @@ -334,8 +334,7 @@ jobs: TemplateFilePath = Join-Path '$(System.DefaultWorkingDirectory)' '$(modulePath)' '$(moduleTestFilePath)' SubscriptionId = '${{ parameters.subscriptionId }}' ManagementGroupId = '${{ parameters.managementGroupId }}' - deploymentName = '$(deploymentName)' - Verbose = $true + DeploymentNames = '$(deploymentNames)' | ConvertFrom-Json } Write-Verbose 'Invoke task with' -Verbose diff --git a/.github/actions/templates/validateModuleDeployment/action.yml b/.github/actions/templates/validateModuleDeployment/action.yml index 2d0fefbb9a..8edd8d20a2 100644 --- a/.github/actions/templates/validateModuleDeployment/action.yml +++ b/.github/actions/templates/validateModuleDeployment/action.yml @@ -271,17 +271,17 @@ runs: $res = New-TemplateDeployment @functionInput -Verbose # Get deployment name - Write-Output ('{0}={1}' -f 'deploymentName', $res.deploymentName) >> $env:GITHUB_OUTPUT + Write-Output ('{0}={1}' -f 'deploymentNames', ($res.deploymentNames | ConvertTo-Json -Compress)) >> $env:GITHUB_OUTPUT # Populate further outputs - $deploymentOutputHash = @{} + $deploymentOutputHashTable = @{} foreach ($outputKey in $res.deploymentOutput.Keys) { Write-Output ('{0}={1}' -f 'outputKey', $res.deploymentOutput[$outputKey].Value) >> $env:GITHUB_OUTPUT - $deploymentOutputHash.add($outputKey, $res.deploymentOutput[$outputKey].Value) + $deploymentOutputHashTable.add($outputKey, $res.deploymentOutput[$outputKey].Value) } - $deploymentOutput = $deploymentOutputHash | ConvertTo-Json -Compress -Depth 100 + $deploymentOutput = $deploymentOutputHashTable | ConvertTo-Json -Compress -Depth 100 Write-Verbose "Deployment output: $deploymentOutput" -Verbose if ($res.ContainsKey('exception')) { @@ -294,7 +294,7 @@ runs: # [Deployment removal] task(s) # ---------------------------- - name: 'Remove deployed resources' - if: ${{ always() && inputs.removeDeployment == 'true' && steps.deploy_step.outputs.deploymentName != '' }} + if: ${{ always() && inputs.removeDeployment == 'true' && steps.deploy_step.outputs.deploymentNames != '' }} uses: azure/powershell@v1 with: azPSVersion: 'latest' @@ -307,9 +307,8 @@ runs: $functionInput = @{ TemplateFilePath = Join-Path $env:GITHUB_WORKSPACE '${{ inputs.templateFilePath }}' - DeploymentName = '${{ steps.deploy_step.outputs.deploymentName }}' + DeploymentNames = '${{ steps.deploy_step.outputs.deploymentNames }}' | ConvertFrom-Json ManagementGroupId = '${{ inputs.managementGroupId }}' - Verbose = $true } Write-Verbose 'Invoke task with' -Verbose diff --git a/docs/wiki/The CI environment - Deployment validation.md b/docs/wiki/The CI environment - Deployment validation.md index a70994cfb9..48839a792e 100644 --- a/docs/wiki/The CI environment - Deployment validation.md +++ b/docs/wiki/The CI environment - Deployment validation.md @@ -54,11 +54,13 @@ The removal step is triggered after the deployment completes. It removes all res However, the removal step can be skipped in case further investigation on the deployed resource is needed. This can be controlled when running the module pipeline leveraging [Module pipeline inputs](./The%20CI%20environment%20-%20Pipeline%20design#module-pipeline-inputs). +> Note: The logic will consider all deployment names used during the deployment step - even those of retries. + ### How it works The removal process will delete all resources created by the deployment. The list of resources is identified by: -1. Recursively fetching the list of resource IDs created in the deployment (identified via the deployment name used). +1. Recursively fetching the list of resource IDs created in the deployment(s) (identified via the deployment names(s) used). 1. Ordering the list based on resource IDs segment count (ensures child resources are removed first. E.g., `storageAccount/blobServices` comes before `storageAccount` as it has one more segments delimited by `/`). 1. Filtering out resources must remain even after the test concluded from the list. This contains, but is not limited to: 1. Resources that are autogenerated by Azure and can cause issues if not controlled (e.g., the Network Watcher resource group that is autogenerated and shared by multiple module tests) diff --git a/utilities/pipelines/resourceDeployment/New-TemplateDeployment.ps1 b/utilities/pipelines/resourceDeployment/New-TemplateDeployment.ps1 index 501dfbaf51..df74dd682a 100644 --- a/utilities/pipelines/resourceDeployment/New-TemplateDeployment.ps1 +++ b/utilities/pipelines/resourceDeployment/New-TemplateDeployment.ps1 @@ -217,6 +217,7 @@ function New-TemplateDeploymentInner { $deploymentScope = Get-ScopeOfTemplateFile -TemplateFilePath $templateFilePath [bool]$Stoploop = $false [int]$retryCount = 1 + $usedDeploymentNames = @() do { # Generate a valid deployment name. Must match ^[-\w\._\(\)]+$ @@ -225,6 +226,7 @@ function New-TemplateDeploymentInner { } while ($deploymentName -notmatch '^[-\w\._\(\)]+$') Write-Verbose "Deploying with deployment name [$deploymentName]" -Verbose + $usedDeploymentNames += $deploymentName $DeploymentInputs['DeploymentName'] = $deploymentName try { @@ -301,8 +303,8 @@ function New-TemplateDeploymentInner { } return @{ - DeploymentName = $deploymentName - Exception = $exceptionMessage + DeploymentNames = $usedDeploymentNames + Exception = $exceptionMessage } } else { throw $PSitem.Exception.Message @@ -322,8 +324,8 @@ function New-TemplateDeploymentInner { Write-Verbose '------' -Verbose Write-Verbose ($res | Out-String) -Verbose return @{ - deploymentName = $deploymentName - deploymentOutput = $res.Outputs + DeploymentNames = $usedDeploymentNames + DeploymentOutput = $res.Outputs } } diff --git a/utilities/pipelines/resourceRemoval/Initialize-DeploymentRemoval.ps1 b/utilities/pipelines/resourceRemoval/Initialize-DeploymentRemoval.ps1 index 493b0edc08..c138aeddce 100644 --- a/utilities/pipelines/resourceRemoval/Initialize-DeploymentRemoval.ps1 +++ b/utilities/pipelines/resourceRemoval/Initialize-DeploymentRemoval.ps1 @@ -56,13 +56,6 @@ function Initialize-DeploymentRemoval { $null = Set-AzContext -Subscription $subscriptionId } - if (-not (Split-Path (Split-Path $templateFilePath -Parent) -LeafBase)) { - # In case of new dependency approach (template is in subfolder) - $moduleName = Split-Path (Split-Path (Split-Path $templateFilePath -Parent) -Parent) -LeafBase - } else { - $moduleName = Split-Path (Split-Path $templateFilePath -Parent) -LeafBase - } - # The initial sequence is a general order-recommendation $removalSequence = @( 'Microsoft.Authorization/locks', @@ -82,39 +75,36 @@ function Initialize-DeploymentRemoval { 'Microsoft.Resources/resourceGroups', 'Microsoft.Compute/virtualMachines' ) - Write-Verbose ('Template file path: [{0}]' -f $templateFilePath) -Verbose - Write-Verbose ('Module name: [{0}]' -f $moduleName) -Verbose - - foreach ($deploymentName in $deploymentNames) { - Write-Verbose ('Handling resource removal with deployment name [{0}]' -f $deploymentName) -Verbose - - ### CODE LOCATION: Add custom removal sequence here - ## Add custom module-specific removal sequence following the example below - # switch ($moduleName) { - # '' { # For example: 'virtualWans', 'automationAccounts' - # $removalSequence += @( - # '', # For example: 'Microsoft.Network/vpnSites', 'Microsoft.OperationalInsights/workspaces/linkedServices' - # '', - # '' - # ) - # break - # } - # } - - # Invoke removal - $inputObject = @{ - DeploymentName = $deploymentName - TemplateFilePath = $templateFilePath - RemovalSequence = $removalSequence - } - if (-not [String]::IsNullOrEmpty($resourceGroupName)) { - $inputObject['resourceGroupName'] = $resourceGroupName - } - if (-not [String]::IsNullOrEmpty($ManagementGroupId)) { - $inputObject['ManagementGroupId'] = $ManagementGroupId - } - Remove-Deployment @inputObject -Verbose + + Write-Verbose ('Handling resource removal with deployment names [{0}]' -f ($deploymentNames -join ', ')) -Verbose + + ### CODE LOCATION: Add custom removal sequence here + ## Add custom module-specific removal sequence following the example below + # $moduleName = Split-Path (Split-Path (Split-Path $templateFilePath -Parent) -Parent) -LeafBase + # switch ($moduleName) { + # '' { # For example: 'virtualWans', 'automationAccounts' + # $removalSequence += @( + # '', # For example: 'Microsoft.Network/vpnSites', 'Microsoft.OperationalInsights/workspaces/linkedServices' + # '', + # '' + # ) + # break + # } + # } + + # Invoke removal + $inputObject = @{ + DeploymentNames = $DeploymentNames + TemplateFilePath = $templateFilePath + RemovalSequence = $removalSequence + } + if (-not [String]::IsNullOrEmpty($resourceGroupName)) { + $inputObject['resourceGroupName'] = $resourceGroupName + } + if (-not [String]::IsNullOrEmpty($ManagementGroupId)) { + $inputObject['ManagementGroupId'] = $ManagementGroupId } + Remove-Deployment @inputObject } end { diff --git a/utilities/pipelines/resourceRemoval/helper/Get-DeploymentTargetResourceList.ps1 b/utilities/pipelines/resourceRemoval/helper/Get-DeploymentTargetResourceList.ps1 index 1e9df42972..c455fe3669 100644 --- a/utilities/pipelines/resourceRemoval/helper/Get-DeploymentTargetResourceList.ps1 +++ b/utilities/pipelines/resourceRemoval/helper/Get-DeploymentTargetResourceList.ps1 @@ -91,7 +91,7 @@ function Get-DeploymentTargetResourceListInner { # Manage nested resources # ########################### foreach ($deployment in ($deploymentTargets | Where-Object { $_ -notmatch '/deployments/' } )) { - Write-Verbose ('Found deployed resource [{0}]' -f $deployment) -Verbose + Write-Verbose ('Found deployed resource [{0}]' -f $deployment) [array]$resultSet += $deployment } @@ -103,23 +103,23 @@ function Get-DeploymentTargetResourceListInner { if ($deployment -match '/resourceGroups/') { # Resource Group Level Child Deployments # ########################################## - Write-Verbose ('Found [resource group] deployment [{0}]' -f $deployment) -Verbose + Write-Verbose ('Found [resource group] deployment [{0}]' -f $deployment) $resourceGroupName = $deployment.split('/resourceGroups/')[1].Split('/')[0] [array]$resultSet += Get-DeploymentTargetResourceListInner -Name $name -Scope 'resourcegroup' -ResourceGroupName $ResourceGroupName } elseif ($deployment -match '/subscriptions/') { # Subscription Level Child Deployments # ######################################## - Write-Verbose ('Found [subscription] deployment [{0}]' -f $deployment) -Verbose + Write-Verbose ('Found [subscription] deployment [{0}]' -f $deployment) [array]$resultSet += Get-DeploymentTargetResourceListInner -Name $name -Scope 'subscription' } elseif ($deployment -match '/managementgroups/') { # Management Group Level Child Deployments # ############################################ - Write-Verbose ('Found [management group] deployment [{0}]' -f $deployment) -Verbose + Write-Verbose ('Found [management group] deployment [{0}]' -f $deployment) [array]$resultSet += Get-DeploymentTargetResourceListInner -Name $name -Scope 'managementgroup' -ManagementGroupId $ManagementGroupId } else { # Tenant Level Child Deployments # ################################## - Write-Verbose ('Found [tenant] deployment [{0}]' -f $deployment) -Verbose + Write-Verbose ('Found [tenant] deployment [{0}]' -f $deployment) [array]$resultSet += Get-DeploymentTargetResourceListInner -Name $name -Scope 'tenant' } } diff --git a/utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1 b/utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1 index 75a9f9e179..a6325482c3 100644 --- a/utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1 +++ b/utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1 @@ -27,8 +27,6 @@ function Invoke-ResourceRemoval { [string] $Type ) - Write-Verbose ('Removing resource [{0}]' -f $resourceId) -Verbose - switch ($type) { 'Microsoft.Insights/diagnosticSettings' { $parentResourceId = $resourceId.Split('/providers/{0}' -f $type)[0] diff --git a/utilities/pipelines/resourceRemoval/helper/Remove-Deployment.ps1 b/utilities/pipelines/resourceRemoval/helper/Remove-Deployment.ps1 index 6204561b12..ea7117c714 100644 --- a/utilities/pipelines/resourceRemoval/helper/Remove-Deployment.ps1 +++ b/utilities/pipelines/resourceRemoval/helper/Remove-Deployment.ps1 @@ -21,8 +21,8 @@ Optional. The maximum times to retry the search for resources via their removal .PARAMETER SearchRetryInterval Optional. The time to wait in between the search for resources via their remove tags -.PARAMETER DeploymentName -Optional. The deployment name to use for the removal +.PARAMETER DeploymentNames +Optional. The deployment names to use for the removal .PARAMETER TemplateFilePath Mandatory. The path to the deployment file @@ -31,9 +31,9 @@ Mandatory. The path to the deployment file Optional. The order of resource types to apply for deletion .EXAMPLE -Remove-Deployment -DeploymentName 'KeyVault' -ResourceGroupName 'validation-rg' -TemplateFilePath 'C:/deploy.json' +Remove-Deployment -DeploymentNames @('KeyVault-t1','KeyVault-t2') -TemplateFilePath 'C:/deploy.json' -Remove a virtual WAN with deployment name 'keyvault-12345' from resource group 'validation-rg' +Remove all resources deployed via the with deployment names 'KeyVault-t1' & 'KeyVault-t2' #> function Remove-Deployment { @@ -46,7 +46,7 @@ function Remove-Deployment { [string] $ManagementGroupId, [Parameter(Mandatory = $true)] - [string] $DeploymentName, + [string[]] $DeploymentNames, [Parameter(Mandatory = $true)] [string] $TemplateFilePath, @@ -87,17 +87,24 @@ function Remove-Deployment { # Fetch deployments # ================= - $deploymentsInputObject = @{ - Name = $deploymentName - Scope = $deploymentScope - } - if (-not [String]::IsNullOrEmpty($resourceGroupName)) { - $deploymentsInputObject['resourceGroupName'] = $resourceGroupName - } - if (-not [String]::IsNullOrEmpty($ManagementGroupId)) { - $deploymentsInputObject['ManagementGroupId'] = $ManagementGroupId + $deployedTargetResources = @() + + foreach ($deploymentName in $DeploymentNames) { + $deploymentsInputObject = @{ + Name = $deploymentName + Scope = $deploymentScope + } + if (-not [String]::IsNullOrEmpty($resourceGroupName)) { + $deploymentsInputObject['resourceGroupName'] = $resourceGroupName + } + if (-not [String]::IsNullOrEmpty($ManagementGroupId)) { + $deploymentsInputObject['ManagementGroupId'] = $ManagementGroupId + } + $deployedTargetResources += Get-DeploymentTargetResourceList @deploymentsInputObject } - [array] $deployedTargetResources = Get-DeploymentTargetResourceList @deploymentsInputObject -Verbose + + [array] $deployedTargetResources = $deployedTargetResources | Select-Object -Unique + Write-Verbose ('Total number of deployment target resources after fetching deployments [{0}]' -f $deployedTargetResources.Count) -Verbose # Pre-Filter & order items @@ -151,7 +158,7 @@ function Remove-Deployment { # ================ if ($resourcesToRemove.Count -gt 0) { if ($PSCmdlet.ShouldProcess(('[{0}] resources' -f (($resourcesToRemove -is [array]) ? $resourcesToRemove.Count : 1)), 'Remove')) { - Remove-ResourceList -ResourcesToRemove $resourcesToRemove -Verbose + Remove-ResourceList -ResourcesToRemove $resourcesToRemove } } else { Write-Verbose 'Found [0] resources to remove' diff --git a/utilities/pipelines/resourceRemoval/helper/Remove-ResourceList.ps1 b/utilities/pipelines/resourceRemoval/helper/Remove-ResourceList.ps1 index ff7dce093c..924c308db6 100644 --- a/utilities/pipelines/resourceRemoval/helper/Remove-ResourceList.ps1 +++ b/utilities/pipelines/resourceRemoval/helper/Remove-ResourceList.ps1 @@ -40,11 +40,11 @@ function Remove-ResourceListInner { if ($alreadyProcessed) { # Skipping - Write-Verbose ('Skipping resource [{0}] of type [{1}] as a parent resource was already processed' -f $resourceName, $resource.type) -Verbose + Write-Verbose ('[/] Skipping resource [{0}] of type [{1}] as a parent resource was already processed' -f $resourceName, $resource.type) -Verbose [array]$processedResources += $resource.resourceId [array]$resourcesToRetry = $resourcesToRetry | Where-Object { $_.resourceId -notmatch $resource.resourceId } } else { - Write-Verbose ('Removing resource [{0}] of type [{1}]' -f $resourceName, $resource.type) -Verbose + Write-Verbose ('[-] Removing resource [{0}] of type [{1}]' -f $resourceName, $resource.type) -Verbose try { if ($PSCmdlet.ShouldProcess(('Resource [{0}]' -f $resource.resourceId), 'Remove')) { Invoke-ResourceRemoval -Type $resource.type -ResourceId $resource.resourceId @@ -107,7 +107,7 @@ function Remove-ResourceList { do { if ($PSCmdlet.ShouldProcess(("[{0}] Resource(s) with a maximum of [$removalRetryLimit] attempts." -f (($resourcesToRetry -is [array]) ? $resourcesToRetry.Count : 1)), 'Remove')) { - $resourcesToRetry = Remove-ResourceListInner -ResourcesToRemove $resourcesToRetry -Verbose + $resourcesToRetry = Remove-ResourceListInner -ResourcesToRemove $resourcesToRetry } else { Remove-ResourceListInner -ResourcesToRemove $resourcesToRemove -WhatIf }