From d71c2310487c046a6d5df416e39ba55251494a5c Mon Sep 17 00:00:00 2001 From: Isabelle Date: Tue, 7 Apr 2026 15:26:12 -0700 Subject: [PATCH 1/2] adding support for soft deleted blobs and blob versions during resource cleanup --- .../scripts/Helpers/Resource-Helpers.ps1 | 53 ++++++++++++++++--- 1 file changed, 47 insertions(+), 6 deletions(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index 5fca15845aeb..12d85df088f7 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -434,11 +434,17 @@ function RemoveStorageAccount($Account) { try { foreach ($container in $containers) { - $blobs = $container | Get-AzStorageBlob - foreach ($blob in $blobs) { - $shouldDelete = EnableBlobDeletion -Blob $blob -Container $container -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName - if ($shouldDelete) { - $deleteNow += $blob + # VLW containers need version-aware cleanup: soft-delete causes deleted blobs to linger + # as non-current versions that block container deletion. See Remove-VlwContainerBlobs. + if (($container | Get-Member 'BlobContainerProperties') -and $container.BlobContainerProperties.HasImmutableStorageWithVersioning) { + Remove-VlwContainerBlobs -Container $container -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName + } else { + $blobs = $container | Get-AzStorageBlob + foreach ($blob in $blobs) { + $shouldDelete = EnableBlobDeletion -Blob $blob -Container $container -StorageAccountName $Account.StorageAccountName -ResourceGroupName $Account.ResourceGroupName + if ($shouldDelete) { + $deleteNow += $blob + } } } } @@ -524,6 +530,41 @@ function EnableBlobDeletion($Blob, $Container, $StorageAccountName, $ResourceGro return $forceBlobDeletion } +# In VLW (Versioned-Level WORM) containers with soft-delete enabled, deleting a blob creates a +# non-current version instead of truly removing it. A standard Get-AzStorageBlob listing can't +# see these leftovers, but they still block container deletion (409 Conflict on the management +# plane DELETE). Listing with -IncludeVersion -IncludeDeleted makes them visible so we can clear +# immutability policies / legal holds and delete each version individually. Multiple passes handle +# new non-current versions that surface after each round of deletions. +function Remove-VlwContainerBlobs($Container, $StorageAccountName, $ResourceGroupName) { + Write-Host "Cleaning VLW container '$($Container.Name)' versions and soft-deleted blobs in account '$StorageAccountName'" + + for ($round = 0; $round -lt 5; $round++) { + $found = $false + $blobs = @($Container | Get-AzStorageBlob -IncludeVersion -IncludeDeleted -ErrorAction SilentlyContinue) + + foreach ($blob in $blobs) { + $found = $true + + # Unconditionally clear legal holds and immutability policies. Errors are expected for + # soft-deleted blobs or blobs that don't have these set. + try { $blob | Set-AzStorageBlobLegalHold -DisableLegalHold | Out-Null } catch { } + try { $blob | Remove-AzStorageBlobImmutabilityPolicy } catch { } + try { + $blob | Remove-AzStorageBlob -Force + } catch { + # Deleting the current version by version ID returns 403 + # (OperationNotAllowedOnRootBlob); fall back to base blob deletion. + try { + Remove-AzStorageBlob -Container $Container.Name -Blob $blob.Name -Context $Container.Context -Force + } catch { } + } + } + + if (-not $found) { break } + } +} + function DoesSubnetOverlap([string]$ipOrCidr, [string]$overlapIp) { [System.Net.IPAddress]$overlapIpAddress = $overlapIp $parsed = $ipOrCidr -split '/' @@ -543,4 +584,4 @@ function DoesSubnetOverlap([string]$ipOrCidr, [string]$overlapIp) { } return $baseIp.Address -eq ($overlapIpAddress.Address -band ([System.Net.IPAddress]$mask).Address) -} +} \ No newline at end of file From c4b3abcd900a15a59485a71e1dedcb87800b2e0a Mon Sep 17 00:00:00 2001 From: Isabelle <141270045+ibrandes@users.noreply.github.com> Date: Wed, 8 Apr 2026 11:22:58 -0700 Subject: [PATCH 2/2] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- eng/common/scripts/Helpers/Resource-Helpers.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/common/scripts/Helpers/Resource-Helpers.ps1 b/eng/common/scripts/Helpers/Resource-Helpers.ps1 index 12d85df088f7..64ea0aeaac28 100644 --- a/eng/common/scripts/Helpers/Resource-Helpers.ps1 +++ b/eng/common/scripts/Helpers/Resource-Helpers.ps1 @@ -537,7 +537,7 @@ function EnableBlobDeletion($Blob, $Container, $StorageAccountName, $ResourceGro # immutability policies / legal holds and delete each version individually. Multiple passes handle # new non-current versions that surface after each round of deletions. function Remove-VlwContainerBlobs($Container, $StorageAccountName, $ResourceGroupName) { - Write-Host "Cleaning VLW container '$($Container.Name)' versions and soft-deleted blobs in account '$StorageAccountName'" + Write-Host "Cleaning VLW container '$($Container.Name)' versions and soft-deleted blobs in account '$StorageAccountName', group: $ResourceGroupName" for ($round = 0; $round -lt 5; $round++) { $found = $false @@ -549,7 +549,7 @@ function Remove-VlwContainerBlobs($Container, $StorageAccountName, $ResourceGrou # Unconditionally clear legal holds and immutability policies. Errors are expected for # soft-deleted blobs or blobs that don't have these set. try { $blob | Set-AzStorageBlobLegalHold -DisableLegalHold | Out-Null } catch { } - try { $blob | Remove-AzStorageBlobImmutabilityPolicy } catch { } + try { $blob | Remove-AzStorageBlobImmutabilityPolicy | Out-Null } catch { } try { $blob | Remove-AzStorageBlob -Force } catch {