[Fixes] Improve removal of resources in a retry scenario

.azuredevops/pipelineTemplates/jobs.validateModuleDeployment.yml

@@ -293,20 +293,20 @@ jobs:
             $res = New-TemplateDeployment @functionInput -Verbose
 
             # Get deployment name
-            $deploymentName = $res.deploymentName
-            Write-Verbose "Deployment name: $deploymentName" -Verbose
-            Write-Host "##vso[task.setvariable variable=deploymentName]$deploymentName"
-            Write-Host "##vso[task.setvariable variable=deploymentName;isOutput=true]$deploymentName"
+            $deploymentNames = $res.DeploymentNames | ConvertTo-Json -Compress
+            Write-Verbose "Deployment name(s) [$deploymentNames]" -Verbose
+            Write-Host "##vso[task.setvariable variable=deploymentNames]$deploymentNames"
+            Write-Host "##vso[task.setvariable variable=deploymentNames;isOutput=true]$deploymentNames"
 
             # Populate further outputs
-            $deploymentOutputHash=@{}
+            $deploymentOutputHashTable=@{}
 
             foreach ($outputKey in $res.deploymentOutput.Keys) {
               Write-Output ('##vso[task.setvariable variable={0}]{1}' -f $outputKey, $res.deploymentOutput[$outputKey].Value)
-              $deploymentOutputHash.add($outputKey,$res.deploymentOutput[$outputKey].Value)
+              $deploymentOutputHashTable.add($outputKey,$res.deploymentOutput[$outputKey].Value)
             }
 
-            $deploymentOutput = $deploymentOutputHash | ConvertTo-Json -Compress -Depth 100
+            $deploymentOutput = $deploymentOutputHashTable | ConvertTo-Json -Compress -Depth 100
             Write-Verbose "Deployment output: $deploymentOutput" -Verbose
 
             if ($res.ContainsKey('exception')) {
@@ -318,7 +318,7 @@ jobs:
       #------------------
       - task: AzurePowerShell@5
         displayName: 'Remove deployed resources via [${{ parameters.serviceConnection }}]'
-        condition: and(succeededOrFailed(), eq('${{ parameters.removeDeployment }}', 'True'), not(eq(variables['deploymentName'],'')), not(startsWith(variables['deploymentName'], 'variables[' )))
+        condition: and(succeededOrFailed(), eq('${{ parameters.removeDeployment }}', 'True'), not(eq(variables['deploymentNames'],'')), not(startsWith(variables['deploymentNames'], 'variables[' )))
         inputs:
           azureSubscription: ${{ parameters.serviceConnection }}
           azurePowerShellVersion: ${{ parameters.azurePowerShellVersion }}
@@ -334,8 +334,7 @@ jobs:
               TemplateFilePath  = Join-Path '$(System.DefaultWorkingDirectory)' '$(modulePath)' '$(moduleTestFilePath)'
               SubscriptionId    = '${{ parameters.subscriptionId }}'
               ManagementGroupId = '${{ parameters.managementGroupId }}'
-              deploymentName    = '$(deploymentName)'
-              Verbose           = $true
+              DeploymentNames   = '$(deploymentNames)' | ConvertFrom-Json
             }
 
             Write-Verbose 'Invoke task with' -Verbose

.github/actions/templates/validateModuleDeployment/action.yml

@@ -271,17 +271,17 @@ runs:
           $res = New-TemplateDeployment @functionInput -Verbose
 
           # Get deployment name
-          Write-Output ('{0}={1}' -f 'deploymentName', $res.deploymentName) >> $env:GITHUB_OUTPUT
+          Write-Output ('{0}={1}' -f 'deploymentNames', ($res.deploymentNames | ConvertTo-Json -Compress)) >> $env:GITHUB_OUTPUT
 
           # Populate further outputs
-          $deploymentOutputHash = @{}
+          $deploymentOutputHashTable = @{}
 
           foreach ($outputKey in $res.deploymentOutput.Keys) {
             Write-Output ('{0}={1}' -f 'outputKey', $res.deploymentOutput[$outputKey].Value) >> $env:GITHUB_OUTPUT
-            $deploymentOutputHash.add($outputKey, $res.deploymentOutput[$outputKey].Value)
+            $deploymentOutputHashTable.add($outputKey, $res.deploymentOutput[$outputKey].Value)
           }
 
-          $deploymentOutput = $deploymentOutputHash | ConvertTo-Json -Compress -Depth 100
+          $deploymentOutput = $deploymentOutputHashTable | ConvertTo-Json -Compress -Depth 100
           Write-Verbose "Deployment output: $deploymentOutput" -Verbose
 
           if ($res.ContainsKey('exception')) {
@@ -294,7 +294,7 @@ runs:
     # [Deployment removal] task(s)
     # ----------------------------
     - name: 'Remove deployed resources'
-      if: ${{ always() && inputs.removeDeployment == 'true' && steps.deploy_step.outputs.deploymentName != '' }}
+      if: ${{ always() && inputs.removeDeployment == 'true' && steps.deploy_step.outputs.deploymentNames != '' }}
       uses: azure/powershell@v1
       with:
         azPSVersion: 'latest'
@@ -307,9 +307,8 @@ runs:
 
           $functionInput = @{
             TemplateFilePath  = Join-Path $env:GITHUB_WORKSPACE '${{ inputs.templateFilePath }}'
-            DeploymentName    = '${{ steps.deploy_step.outputs.deploymentName }}'
+            DeploymentNames   = '${{ steps.deploy_step.outputs.deploymentNames }}' | ConvertFrom-Json
             ManagementGroupId = '${{ inputs.managementGroupId }}'
-            Verbose           = $true
           }
 
           Write-Verbose 'Invoke task with' -Verbose

docs/wiki/The CI environment - Deployment validation.md

@@ -54,11 +54,13 @@ The removal step is triggered after the deployment completes. It removes all res
 
 However, the removal step can be skipped in case further investigation on the deployed resource is needed. This can be controlled when running the module pipeline leveraging [Module pipeline inputs](./The%20CI%20environment%20-%20Pipeline%20design#module-pipeline-inputs).
 
+> Note: The logic will consider all deployment names used during the deployment step - even those of retries.
+
 ### How it works
 
 The removal process will delete all resources created by the deployment. The list of resources is identified by:
 
-1. Recursively fetching the list of resource IDs created in the deployment (identified via the deployment name used).
+1. Recursively fetching the list of resource IDs created in the deployment(s) (identified via the deployment names(s) used).
 1. Ordering the list based on resource IDs segment count (ensures child resources are removed first. E.g., `storageAccount/blobServices` comes before `storageAccount` as it has one more segments delimited by `/`).
 1. Filtering out resources must remain even after the test concluded from the list. This contains, but is not limited to:
    1. Resources that are autogenerated by Azure and can cause issues if not controlled (e.g., the Network Watcher resource group that is autogenerated and shared by multiple module tests)

utilities/pipelines/resourceDeployment/New-TemplateDeployment.ps1

@@ -217,6 +217,7 @@ function New-TemplateDeploymentInner {
         $deploymentScope = Get-ScopeOfTemplateFile -TemplateFilePath $templateFilePath
         [bool]$Stoploop = $false
         [int]$retryCount = 1
+        $usedDeploymentNames = @()
 
         do {
             # Generate a valid deployment name. Must match ^[-\w\._\(\)]+$
@@ -225,6 +226,7 @@ function New-TemplateDeploymentInner {
             } while ($deploymentName -notmatch '^[-\w\._\(\)]+$')
 
             Write-Verbose "Deploying with deployment name [$deploymentName]" -Verbose
+            $usedDeploymentNames += $deploymentName
             $DeploymentInputs['DeploymentName'] = $deploymentName
 
             try {
@@ -301,8 +303,8 @@ function New-TemplateDeploymentInner {
                         }
 
                         return @{
-                            DeploymentName = $deploymentName
-                            Exception      = $exceptionMessage
+                            DeploymentNames = $usedDeploymentNames
+                            Exception       = $exceptionMessage
                         }
                     } else {
                         throw $PSitem.Exception.Message
@@ -322,8 +324,8 @@ function New-TemplateDeploymentInner {
         Write-Verbose '------' -Verbose
         Write-Verbose ($res | Out-String) -Verbose
         return @{
-            deploymentName   = $deploymentName
-            deploymentOutput = $res.Outputs
+            DeploymentNames  = $usedDeploymentNames
+            DeploymentOutput = $res.Outputs
         }
     }
 

utilities/pipelines/resourceRemoval/Initialize-DeploymentRemoval.ps1

@@ -56,13 +56,6 @@ function Initialize-DeploymentRemoval {
             $null = Set-AzContext -Subscription $subscriptionId
         }
 
-        if (-not (Split-Path (Split-Path $templateFilePath -Parent) -LeafBase)) {
-            # In case of new dependency approach (template is in subfolder)
-            $moduleName = Split-Path (Split-Path (Split-Path $templateFilePath -Parent) -Parent) -LeafBase
-        } else {
-            $moduleName = Split-Path (Split-Path $templateFilePath -Parent) -LeafBase
-        }
-
         # The initial sequence is a general order-recommendation
         $removalSequence = @(
             'Microsoft.Authorization/locks',
@@ -82,39 +75,36 @@ function Initialize-DeploymentRemoval {
             'Microsoft.Resources/resourceGroups',
             'Microsoft.Compute/virtualMachines'
         )
-        Write-Verbose ('Template file path: [{0}]' -f $templateFilePath) -Verbose
-        Write-Verbose ('Module name: [{0}]' -f $moduleName) -Verbose
-
-        foreach ($deploymentName in $deploymentNames) {
-            Write-Verbose ('Handling resource removal with deployment name [{0}]' -f $deploymentName) -Verbose
-
-            ### CODE LOCATION: Add custom removal sequence here
-            ## Add custom module-specific removal sequence following the example below
-            # switch ($moduleName) {
-            #     '<moduleName01>' {                # For example: 'virtualWans', 'automationAccounts'
-            #         $removalSequence += @(
-            #             '<resourceType01>',       # For example: 'Microsoft.Network/vpnSites', 'Microsoft.OperationalInsights/workspaces/linkedServices'
-            #             '<resourceType02>',
-            #             '<resourceType03>'
-            #         )
-            #         break
-            #     }
-            # }
-
-            # Invoke removal
-            $inputObject = @{
-                DeploymentName   = $deploymentName
-                TemplateFilePath = $templateFilePath
-                RemovalSequence  = $removalSequence
-            }
-            if (-not [String]::IsNullOrEmpty($resourceGroupName)) {
-                $inputObject['resourceGroupName'] = $resourceGroupName
-            }
-            if (-not [String]::IsNullOrEmpty($ManagementGroupId)) {
-                $inputObject['ManagementGroupId'] = $ManagementGroupId
-            }
-            Remove-Deployment @inputObject -Verbose
+
+        Write-Verbose ('Handling resource removal with deployment names [{0}]' -f ($deploymentNames -join ', ')) -Verbose
+
+        ### CODE LOCATION: Add custom removal sequence here
+        ## Add custom module-specific removal sequence following the example below
+        # $moduleName = Split-Path (Split-Path (Split-Path $templateFilePath -Parent) -Parent) -LeafBase
+        # switch ($moduleName) {
+        #     '<moduleName01>' {                # For example: 'virtualWans', 'automationAccounts'
+        #         $removalSequence += @(
+        #             '<resourceType01>',       # For example: 'Microsoft.Network/vpnSites', 'Microsoft.OperationalInsights/workspaces/linkedServices'
+        #             '<resourceType02>',
+        #             '<resourceType03>'
+        #         )
+        #         break
+        #     }
+        # }
+
+        # Invoke removal
+        $inputObject = @{
+            DeploymentNames  = $DeploymentNames
+            TemplateFilePath = $templateFilePath
+            RemovalSequence  = $removalSequence
+        }
+        if (-not [String]::IsNullOrEmpty($resourceGroupName)) {
+            $inputObject['resourceGroupName'] = $resourceGroupName
+        }
+        if (-not [String]::IsNullOrEmpty($ManagementGroupId)) {
+            $inputObject['ManagementGroupId'] = $ManagementGroupId
         }
+        Remove-Deployment @inputObject
     }
 
     end {

utilities/pipelines/resourceRemoval/helper/Get-DeploymentTargetResourceList.ps1

@@ -91,7 +91,7 @@ function Get-DeploymentTargetResourceListInner {
     # Manage nested resources #
     ###########################
     foreach ($deployment in ($deploymentTargets | Where-Object { $_ -notmatch '/deployments/' } )) {
-        Write-Verbose ('Found deployed resource [{0}]' -f $deployment) -Verbose
+        Write-Verbose ('Found deployed resource [{0}]' -f $deployment)
         [array]$resultSet += $deployment
     }
 
@@ -103,23 +103,23 @@ function Get-DeploymentTargetResourceListInner {
         if ($deployment -match '/resourceGroups/') {
             # Resource Group Level Child Deployments #
             ##########################################
-            Write-Verbose ('Found [resource group] deployment [{0}]' -f $deployment) -Verbose
+            Write-Verbose ('Found [resource group] deployment [{0}]' -f $deployment)
             $resourceGroupName = $deployment.split('/resourceGroups/')[1].Split('/')[0]
             [array]$resultSet += Get-DeploymentTargetResourceListInner -Name $name -Scope 'resourcegroup' -ResourceGroupName $ResourceGroupName
         } elseif ($deployment -match '/subscriptions/') {
             # Subscription Level Child Deployments #
             ########################################
-            Write-Verbose ('Found [subscription] deployment [{0}]' -f $deployment) -Verbose
+            Write-Verbose ('Found [subscription] deployment [{0}]' -f $deployment)
             [array]$resultSet += Get-DeploymentTargetResourceListInner -Name $name -Scope 'subscription'
         } elseif ($deployment -match '/managementgroups/') {
             # Management Group Level Child Deployments #
             ############################################
-            Write-Verbose ('Found [management group] deployment [{0}]' -f $deployment) -Verbose
+            Write-Verbose ('Found [management group] deployment [{0}]' -f $deployment)
             [array]$resultSet += Get-DeploymentTargetResourceListInner -Name $name -Scope 'managementgroup' -ManagementGroupId $ManagementGroupId
         } else {
             # Tenant Level Child Deployments #
             ##################################
-            Write-Verbose ('Found [tenant] deployment [{0}]' -f $deployment) -Verbose
+            Write-Verbose ('Found [tenant] deployment [{0}]' -f $deployment)
             [array]$resultSet += Get-DeploymentTargetResourceListInner -Name $name -Scope 'tenant'
         }
     }

utilities/pipelines/resourceRemoval/helper/Invoke-ResourceRemoval.ps1

@@ -27,8 +27,6 @@ function Invoke-ResourceRemoval {
         [string] $Type
     )
 
-    Write-Verbose ('Removing resource [{0}]' -f $resourceId) -Verbose
-
     switch ($type) {
         'Microsoft.Insights/diagnosticSettings' {
             $parentResourceId = $resourceId.Split('/providers/{0}' -f $type)[0]

utilities/pipelines/resourceRemoval/helper/Remove-Deployment.ps1

@@ -21,8 +21,8 @@ Optional. The maximum times to retry the search for resources via their removal
 .PARAMETER SearchRetryInterval
 Optional. The time to wait in between the search for resources via their remove tags
 
-.PARAMETER DeploymentName
-Optional. The deployment name to use for the removal
+.PARAMETER DeploymentNames
+Optional. The deployment names to use for the removal
 
 .PARAMETER TemplateFilePath
 Mandatory. The path to the deployment file
@@ -31,9 +31,9 @@ Mandatory. The path to the deployment file
 Optional. The order of resource types to apply for deletion
 
 .EXAMPLE
-Remove-Deployment -DeploymentName 'KeyVault' -ResourceGroupName 'validation-rg' -TemplateFilePath 'C:/deploy.json'
+Remove-Deployment -DeploymentNames @('KeyVault-t1','KeyVault-t2') -TemplateFilePath 'C:/deploy.json'
 
-Remove a virtual WAN with deployment name 'keyvault-12345' from resource group 'validation-rg'
+Remove all resources deployed via the with deployment names 'KeyVault-t1' & 'KeyVault-t2'
 #>
 function Remove-Deployment {
 
@@ -46,7 +46,7 @@ function Remove-Deployment {
         [string] $ManagementGroupId,
 
         [Parameter(Mandatory = $true)]
-        [string] $DeploymentName,
+        [string[]] $DeploymentNames,
 
         [Parameter(Mandatory = $true)]
         [string] $TemplateFilePath,
@@ -87,17 +87,24 @@ function Remove-Deployment {
 
         # Fetch deployments
         # =================
-        $deploymentsInputObject = @{
-            Name  = $deploymentName
-            Scope = $deploymentScope
-        }
-        if (-not [String]::IsNullOrEmpty($resourceGroupName)) {
-            $deploymentsInputObject['resourceGroupName'] = $resourceGroupName
-        }
-        if (-not [String]::IsNullOrEmpty($ManagementGroupId)) {
-            $deploymentsInputObject['ManagementGroupId'] = $ManagementGroupId
+        $deployedTargetResources = @()
+
+        foreach ($deploymentName in $DeploymentNames) {
+            $deploymentsInputObject = @{
+                Name  = $deploymentName
+                Scope = $deploymentScope
+            }
+            if (-not [String]::IsNullOrEmpty($resourceGroupName)) {
+                $deploymentsInputObject['resourceGroupName'] = $resourceGroupName
+            }
+            if (-not [String]::IsNullOrEmpty($ManagementGroupId)) {
+                $deploymentsInputObject['ManagementGroupId'] = $ManagementGroupId
+            }
+            $deployedTargetResources += Get-DeploymentTargetResourceList @deploymentsInputObject
         }
-        [array] $deployedTargetResources = Get-DeploymentTargetResourceList @deploymentsInputObject -Verbose
+
+        [array] $deployedTargetResources = $deployedTargetResources | Select-Object -Unique
+
         Write-Verbose ('Total number of deployment target resources after fetching deployments [{0}]' -f $deployedTargetResources.Count) -Verbose
 
         # Pre-Filter & order items
@@ -151,7 +158,7 @@ function Remove-Deployment {
         # ================
         if ($resourcesToRemove.Count -gt 0) {
             if ($PSCmdlet.ShouldProcess(('[{0}] resources' -f (($resourcesToRemove -is [array]) ? $resourcesToRemove.Count : 1)), 'Remove')) {
-                Remove-ResourceList -ResourcesToRemove $resourcesToRemove -Verbose
+                Remove-ResourceList -ResourcesToRemove $resourcesToRemove
             }
         } else {
             Write-Verbose 'Found [0] resources to remove'

utilities/pipelines/resourceRemoval/helper/Remove-ResourceList.ps1

@@ -40,11 +40,11 @@ function Remove-ResourceListInner {
 
             if ($alreadyProcessed) {
                 # Skipping
-                Write-Verbose ('Skipping resource [{0}] of type [{1}] as a parent resource was already processed' -f $resourceName, $resource.type) -Verbose
+                Write-Verbose ('[/] Skipping resource [{0}] of type [{1}] as a parent resource was already processed' -f $resourceName, $resource.type) -Verbose
                 [array]$processedResources += $resource.resourceId
                 [array]$resourcesToRetry = $resourcesToRetry | Where-Object { $_.resourceId -notmatch $resource.resourceId }
             } else {
-                Write-Verbose ('Removing resource [{0}] of type [{1}]' -f $resourceName, $resource.type) -Verbose
+                Write-Verbose ('[-] Removing resource [{0}] of type [{1}]' -f $resourceName, $resource.type) -Verbose
                 try {
                     if ($PSCmdlet.ShouldProcess(('Resource [{0}]' -f $resource.resourceId), 'Remove')) {
                         Invoke-ResourceRemoval -Type $resource.type -ResourceId $resource.resourceId
@@ -107,7 +107,7 @@ function Remove-ResourceList {
 
     do {
         if ($PSCmdlet.ShouldProcess(("[{0}] Resource(s) with a maximum of [$removalRetryLimit] attempts." -f (($resourcesToRetry -is [array]) ? $resourcesToRetry.Count : 1)), 'Remove')) {
-            $resourcesToRetry = Remove-ResourceListInner -ResourcesToRemove $resourcesToRetry -Verbose
+            $resourcesToRetry = Remove-ResourceListInner -ResourcesToRemove $resourcesToRetry
         } else {
             Remove-ResourceListInner -ResourcesToRemove $resourcesToRemove -WhatIf
         }