From fdebbda583e78a00a739f481cd1a6ac0d987dcdd Mon Sep 17 00:00:00 2001 From: Marius Date: Wed, 15 Dec 2021 19:50:50 +0100 Subject: [PATCH 1/7] Things --- utilities/tools/ConvertTo-ARMTemplate.ps1 | 95 +++++++++++++++++------ 1 file changed, 72 insertions(+), 23 deletions(-) diff --git a/utilities/tools/ConvertTo-ARMTemplate.ps1 b/utilities/tools/ConvertTo-ARMTemplate.ps1 index ca5f4ae519..15af4512a0 100644 --- a/utilities/tools/ConvertTo-ARMTemplate.ps1 +++ b/utilities/tools/ConvertTo-ARMTemplate.ps1 @@ -24,11 +24,14 @@ Converts bicep modules to json based ARM template, cleaning up all bicep files a #> [CmdletBinding(SupportsShouldProcess)] param ( - [Parameter(Mandatory)] - [string] $Path, + [Parameter()] + [string] $Path = (Get-Location).Path, + + [Parameter()] + [switch] $CleanUp, [Parameter()] - [switch] $CleanUp + [switch] $SkipChildResources ) #region Helper functions @@ -62,43 +65,89 @@ function Remove-JSONMetadata { } } +<# +.SYNOPSIS +Gets the folder objects for the child resources of the provided parent folder. + +.DESCRIPTION +Gets the folder objects for the child resources of the provided parent folder. +This will check for the existence of the child resources and if found, will return the folder objects. +The criteria for the child resource folder is that the folder contains a 'deploy.json' file. + +.PARAMETER Path +Path to the parent resource folder. + +.EXAMPLE +Get-ChildResourceFolder -Path 'C:\Repos\Azure\ResourceModules\arm\Microsoft.Storage\storageAccounts' + +Mode LastWriteTime Length Name +---- ------------- ------ ---- +l---- 15.12.2021 15:27 blobServices +l---- 15.12.2021 15:27 fileServices +l---- 15.12.2021 15:27 managementPolicies +l---- 15.12.2021 15:27 queueServices +l---- 15.12.2021 15:27 tableServices + +Gets the folder objects for the child resources of the provided parent folder. + +#> +function Get-ChildResourceFolder { + + [CmdletBinding()] + param ( + [Parameter(Mandatory = $true)] + [string] $Path + ) + + return Get-ChildItem -Path $Path -Filter 'deploy.json' -Recurse -Depth 1 -Force | Select-Object -ExpandProperty 'Directory' | Get-Item + +} + #endregion $rootPath = Get-Item -Path $Path | Select-Object -ExpandProperty 'FullName' $armFolderPath = Join-Path -Path $rootPath -ChildPath 'arm' -# Get all bicep files -$bicepFiles = Get-ChildItem -Path $armFolderPath -Filter 'deploy.bicep' -Recurse -Force + +# Remove existing json files +Write-Verbose "Removing existing deploy.json files" +Get-ChildItem -Path $armFolderPath -Filter 'deploy.json' -Recurse -Force | ForEach-Object { + if ($PSCmdlet.ShouldProcess("File in path [$($_.FullName)]", 'Remove')) { + Remove-Item -Path $_.FullName -Force + } +} +Write-Verbose "$bicepFilePath - Removing existing deploy.json files - Done" + +# Process all deploy.bicep files +$bicepFiles = Get-ChildItem -Path $armFolderPath -Filter 'deploy.bicep' -Recurse -Force -Depth 2 Write-Verbose "Convert bicep to json - $($bicepFiles.count) files" foreach ($bicepFile in $bicepFiles) { $bicepFilePath = $bicepFile.FullName Write-Verbose "$bicepFilePath - Processing" $moduleFolderPath = $bicepFile.Directory.FullName Write-Verbose "$bicepFilePath - ModuleFolderPath - $moduleFolderPath" + $bicepFolderPath = Join-Path -Path $moduleFolderPath -ChildPath '.bicep' $JSONFilePath = Join-Path -Path $moduleFolderPath -ChildPath 'deploy.json' Write-Verbose "$bicepFilePath - JSONFilePath - $JSONFilePath" - $bicepFolderPath = Join-Path -Path $moduleFolderPath -ChildPath '.bicep' - # Remove existing json files - Write-Verbose "$bicepFilePath - Removing deploy.json" - if (Test-Path -Path $JSONFilePath) { - if ($PSCmdlet.ShouldProcess("File in path [$bicepFilePath]", 'Remove')) { - Remove-Item -Path $JSONFilePath -Force -Verbose - } - Write-Verbose "$bicepFilePath - Removing deploy.json - Done" + + # Convert bicep to json + Write-Verbose "$bicepFilePath - Convert bicep to json" + $BicepFilesToConvert = $bicepFile + if ($SkipChildResources) { + } else { - Write-Verbose "$bicepFilePath - Removing deploy.json - Skipped - Nothing to delete" + Get-ChildItem -Path $bicepFolderPath -Filter 'deploy.bicep' -Recurse -Force } - # Convert bicep to json - Write-Verbose "$bicepFilePath - Convert to json" if ($PSCmdlet.ShouldProcess("File in path [$bicepFilePath]", 'Convert')) { - az bicep build --file $bicepFilePath --outfile $JSONFilePath + Invoke-Expression "az bicep build --file '$bicepFilePath' --outfile '$JSONFilePath'" } - Write-Verbose "$bicepFilePath - Convert to json - Done" + Write-Verbose "$bicepFilePath - Convert bicep to json - Done" + - # Cleanup json, remove metadata property in json - Write-Verbose "$bicepFilePath - Clean up json" + # Remove Bicep metadata from json + Write-Verbose "$bicepFilePath - Remove Bicep metadata from json" if (Test-Path -Path $JSONFilePath) { $JSONFileContent = Get-Content -Path $JSONFilePath $JSONObj = $JSONFileContent | ConvertFrom-Json @@ -107,14 +156,14 @@ foreach ($bicepFile in $bicepFiles) { if ($PSCmdlet.ShouldProcess("File in path [$JSONFilePath]", 'Overwrite')) { Set-Content -Value $JSONFileContent -Path $JSONFilePath } - Write-Verbose "$bicepFilePath - Clean up json - Done" + Write-Verbose "$bicepFilePath - Remove Bicep metadata from json - Done" } else { - Write-Verbose "$bicepFilePath - Clean up json - Skipped - File not found (deploy.json)" + Write-Verbose "$bicepFilePath - Remove Bicep metadata from json - Skipped - File not found (deploy.json)" } # Remove bicep files and folders - Write-Verbose "$bicepFilePath - Clean up bicep files" if ($CleanUp) { + Write-Verbose "$bicepFilePath - Clean up bicep files" if ($PSCmdlet.ShouldProcess("File in path [$bicepFilePath]", 'Remove')) { Remove-Item -Path $bicepFilePath -Force } From e2e0d31a5439a6d8999f88df301c7fcac8e9b86f Mon Sep 17 00:00:00 2001 From: Marius Date: Wed, 15 Dec 2021 23:57:04 +0100 Subject: [PATCH 2/7] Update --- utilities/tools/ConvertTo-ARMTemplate.ps1 | 208 +++++++++------------- 1 file changed, 81 insertions(+), 127 deletions(-) diff --git a/utilities/tools/ConvertTo-ARMTemplate.ps1 b/utilities/tools/ConvertTo-ARMTemplate.ps1 index 15af4512a0..fad9bc3f9a 100644 --- a/utilities/tools/ConvertTo-ARMTemplate.ps1 +++ b/utilities/tools/ConvertTo-ARMTemplate.ps1 @@ -31,153 +31,106 @@ param ( [switch] $CleanUp, [Parameter()] - [switch] $SkipChildResources + [switch] $KeepChildResources ) -#region Helper functions - -<# -.SYNOPSIS -A function to recursively remove 'metadata' property from a provided object. -This object is expected to be an ARM template converted to a PowerShell custom object. -The function uses the object reference rather than recreating/copying the object. - -.PARAMETER TemplateObject -Mandatory. The ARM template converted to a PowerShell custom object. - -.EXAMPLE -$JSONFileContent = Get-Content -Path $JSONFilePath -$JSONObj = $JSONFileContent | ConvertFrom-Json -Remove-JSONMetadata -TemplateObject $JSONObj - -Reads content from a ARM/JSON file, converts it to a PSCustomObject and removes 'metadata' property under the template and recursively on all nested deployments. - -#> -function Remove-JSONMetadata { - [CmdletBinding()] - param ( - [Parameter(Mandatory)] - [psobject] $TemplateObject - ) - $TemplateObject.PSObject.Properties.Remove('metadata') - $TemplateObject.resources | Where-Object { $_.type -eq 'Microsoft.Resources/deployments' } | ForEach-Object { - Remove-JSONMetadata -TemplateObject $_.properties.template - } -} - -<# -.SYNOPSIS -Gets the folder objects for the child resources of the provided parent folder. - -.DESCRIPTION -Gets the folder objects for the child resources of the provided parent folder. -This will check for the existence of the child resources and if found, will return the folder objects. -The criteria for the child resource folder is that the folder contains a 'deploy.json' file. - -.PARAMETER Path -Path to the parent resource folder. - -.EXAMPLE -Get-ChildResourceFolder -Path 'C:\Repos\Azure\ResourceModules\arm\Microsoft.Storage\storageAccounts' - -Mode LastWriteTime Length Name ----- ------------- ------ ---- -l---- 15.12.2021 15:27 blobServices -l---- 15.12.2021 15:27 fileServices -l---- 15.12.2021 15:27 managementPolicies -l---- 15.12.2021 15:27 queueServices -l---- 15.12.2021 15:27 tableServices - -Gets the folder objects for the child resources of the provided parent folder. - -#> -function Get-ChildResourceFolder { - - [CmdletBinding()] - param ( - [Parameter(Mandatory = $true)] - [string] $Path - ) - - return Get-ChildItem -Path $Path -Filter 'deploy.json' -Recurse -Depth 1 -Force | Select-Object -ExpandProperty 'Directory' | Get-Item - -} - -#endregion - $rootPath = Get-Item -Path $Path | Select-Object -ExpandProperty 'FullName' $armFolderPath = Join-Path -Path $rootPath -ChildPath 'arm' - -# Remove existing json files -Write-Verbose "Removing existing deploy.json files" +#region Remove existing json files +Write-Verbose 'Removing existing deploy.json files' Get-ChildItem -Path $armFolderPath -Filter 'deploy.json' -Recurse -Force | ForEach-Object { if ($PSCmdlet.ShouldProcess("File in path [$($_.FullName)]", 'Remove')) { Remove-Item -Path $_.FullName -Force } } -Write-Verbose "$bicepFilePath - Removing existing deploy.json files - Done" - -# Process all deploy.bicep files -$bicepFiles = Get-ChildItem -Path $armFolderPath -Filter 'deploy.bicep' -Recurse -Force -Depth 2 -Write-Verbose "Convert bicep to json - $($bicepFiles.count) files" -foreach ($bicepFile in $bicepFiles) { - $bicepFilePath = $bicepFile.FullName - Write-Verbose "$bicepFilePath - Processing" - $moduleFolderPath = $bicepFile.Directory.FullName - Write-Verbose "$bicepFilePath - ModuleFolderPath - $moduleFolderPath" - $bicepFolderPath = Join-Path -Path $moduleFolderPath -ChildPath '.bicep' - $JSONFilePath = Join-Path -Path $moduleFolderPath -ChildPath 'deploy.json' - Write-Verbose "$bicepFilePath - JSONFilePath - $JSONFilePath" - - - # Convert bicep to json - Write-Verbose "$bicepFilePath - Convert bicep to json" - $BicepFilesToConvert = $bicepFile - if ($SkipChildResources) { - - } else { - Get-ChildItem -Path $bicepFolderPath -Filter 'deploy.bicep' -Recurse -Force - } +Write-Verbose 'Removing existing deploy.json files - Done' +#endregion + +if ($KeepChildResources) { + $BicepFilesToConvert = Get-ChildItem -Path $armFolderPath -Filter 'deploy.bicep' -Recurse -Force +} else { + $BicepFilesToConvert = Get-ChildItem -Path $armFolderPath -Filter 'deploy.bicep' -Recurse -Force -Depth 2 +} - if ($PSCmdlet.ShouldProcess("File in path [$bicepFilePath]", 'Convert')) { - Invoke-Expression "az bicep build --file '$bicepFilePath' --outfile '$JSONFilePath'" +#region Convert bicep files to json +Write-Verbose "Convert bicep files to json - Processing [$($BicepFilesToConvert.count)] files" +if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] file(s) in path [$armFolderPath]", 'Convert')) { + $BicepFilesToConvert | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { + Invoke-Expression "az bicep build --file '$_'" } - Write-Verbose "$bicepFilePath - Convert bicep to json - Done" +} +Write-Verbose 'Convert bicep files to json - Done' +#endregion +#region Remove Bicep metadata from json +Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Processing [$($BicepFilesToConvert.count)] files" +if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] file(s) in path [$armFolderPath]", 'Modify')) { + $BicepFilesToConvert | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { + + function Remove-JSONMetadata { + <# + .SYNOPSIS + A function to recursively remove 'metadata' property from a provided object. + This object is expected to be an ARM template converted to a PowerShell custom object. + The function uses the object reference rather than recreating/copying the object. + + .PARAMETER TemplateObject + Mandatory. The ARM template converted to a PowerShell custom object. + + .EXAMPLE + $JSONFileContent = Get-Content -Path $JSONFilePath + $JSONObj = $JSONFileContent | ConvertFrom-Json + Remove-JSONMetadata -TemplateObject $JSONObj + + Reads content from a ARM/JSON file, converts it to a PSCustomObject and removes 'metadata' property under the template and recursively on all nested deployments. + + #> + + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [psobject] $TemplateObject + ) + $TemplateObject.PSObject.Properties.Remove('metadata') + $TemplateObject.resources | Where-Object { $_.type -eq 'Microsoft.Resources/deployments' } | ForEach-Object { + Remove-JSONMetadata -TemplateObject $_.properties.template + } + } - # Remove Bicep metadata from json - Write-Verbose "$bicepFilePath - Remove Bicep metadata from json" - if (Test-Path -Path $JSONFilePath) { - $JSONFileContent = Get-Content -Path $JSONFilePath - $JSONObj = $JSONFileContent | ConvertFrom-Json - Remove-JSONMetadata -TemplateObject $JSONObj - $JSONFileContent = $JSONObj | ConvertTo-Json -Depth 100 - if ($PSCmdlet.ShouldProcess("File in path [$JSONFilePath]", 'Overwrite')) { + $moduleFolderPath = $_.Directory.FullName + $JSONFilePath = Join-Path -Path $moduleFolderPath -ChildPath 'deploy.json' + if (Test-Path -Path $JSONFilePath) { + $JSONFileContent = Get-Content -Path $JSONFilePath + $JSONObj = $JSONFileContent | ConvertFrom-Json + Remove-JSONMetadata -TemplateObject $JSONObj + $JSONFileContent = $JSONObj | ConvertTo-Json -Depth 100 Set-Content -Value $JSONFileContent -Path $JSONFilePath + Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Done" + } else { + Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Skipped - File not found (deploy.json)" } - Write-Verbose "$bicepFilePath - Remove Bicep metadata from json - Done" - } else { - Write-Verbose "$bicepFilePath - Remove Bicep metadata from json - Skipped - File not found (deploy.json)" } +} +#endregion - # Remove bicep files and folders - if ($CleanUp) { - Write-Verbose "$bicepFilePath - Clean up bicep files" - if ($PSCmdlet.ShouldProcess("File in path [$bicepFilePath]", 'Remove')) { - Remove-Item -Path $bicepFilePath -Force - } - Write-Verbose "$bicepFilePath - Clean up bicep files - Removed deploy.bicep" - if (Test-Path -Path $bicepFolderPath) { - if ($PSCmdlet.ShouldProcess("File in path [$bicepFolderPath]", 'Remove')) { - Remove-Item -Path $bicepFolderPath -Recurse -Force - } - Write-Verbose "$bicepFilePath - Clean up bicep files - Removed .bicep folder" - } - } +#region Remove bicep files and folders +if ($CleanUp) { + Write-Verbose 'Remove bicep files and folders' + + Write-Verbose 'Remove bicep files and folders - Remove .bicep folders' + $dotBicepFolders = Get-ChildItem -Path $armFolderPath -Filter '.bicep' -Recurse -Force -Directory + $dotBicepFolders | Remove-Item -Recurse -Force -WhatIf:$WhatIfPreference + Write-Verbose 'Remove bicep files and folders - Remove .bicep folders - Done' + + Write-Verbose 'Remove bicep files and folders - Remove all *.bicep files' + $BicepFilesToRemove = Get-ChildItem -Path $armFolderPath -Filter '*.bicep' -Recurse -Force + $BicepFilesToRemove | Remove-Item -Force -WhatIf:$WhatIfPreference + Write-Verbose 'Remove bicep files and folders - Remove all *.bicep files - Done' } +#endregion -# Replace .bicep with .json in workflow files +#region Replace .bicep with .json in workflow files $workflowFolderPath = Join-Path -Path $rootPath -ChildPath '.github\workflows' $workflowFiles = Get-ChildItem -Path $workflowFolderPath -Filter 'ms.*.yml' -File -Force Write-Verbose "Update workflow files - $($workflowFiles.count) files" @@ -191,3 +144,4 @@ foreach ($workflowFile in $workflowFiles) { Write-Verbose "$workflowFile - Processing - Done" } Write-Verbose "Update workflow files - $($workflowFiles.count) files - Done" +#endregion From ed557c9077f5c8dc0628c2d40b9e853eb46ceeb9 Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 16 Dec 2021 00:20:28 +0100 Subject: [PATCH 3/7] Updates --- utilities/tools/ConvertTo-ARMTemplate.ps1 | 50 +++++++++++------------ 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/utilities/tools/ConvertTo-ARMTemplate.ps1 b/utilities/tools/ConvertTo-ARMTemplate.ps1 index fad9bc3f9a..cbe21faa71 100644 --- a/utilities/tools/ConvertTo-ARMTemplate.ps1 +++ b/utilities/tools/ConvertTo-ARMTemplate.ps1 @@ -36,28 +36,23 @@ param ( $rootPath = Get-Item -Path $Path | Select-Object -ExpandProperty 'FullName' $armFolderPath = Join-Path -Path $rootPath -ChildPath 'arm' - -#region Remove existing json files -Write-Verbose 'Removing existing deploy.json files' -Get-ChildItem -Path $armFolderPath -Filter 'deploy.json' -Recurse -Force | ForEach-Object { - if ($PSCmdlet.ShouldProcess("File in path [$($_.FullName)]", 'Remove')) { - Remove-Item -Path $_.FullName -Force - } -} -Write-Verbose 'Removing existing deploy.json files - Done' -#endregion - if ($KeepChildResources) { $BicepFilesToConvert = Get-ChildItem -Path $armFolderPath -Filter 'deploy.bicep' -Recurse -Force } else { $BicepFilesToConvert = Get-ChildItem -Path $armFolderPath -Filter 'deploy.bicep' -Recurse -Force -Depth 2 } +#region Remove existing json files +Write-Verbose 'Removing existing deploy.json files' +Get-ChildItem -Path $armFolderPath -Filter 'deploy.json' -Recurse -Force | Remove-Item -Force +Write-Verbose 'Removing existing deploy.json files - Done' +#endregion + #region Convert bicep files to json Write-Verbose "Convert bicep files to json - Processing [$($BicepFilesToConvert.count)] files" -if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] file(s) in path [$armFolderPath]", 'Convert')) { +if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] bicep file(s) in path [$armFolderPath]", 'Convert')) { $BicepFilesToConvert | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { - Invoke-Expression "az bicep build --file '$_'" + Invoke-Expression -Command "az bicep build --file '$_'" } } Write-Verbose 'Convert bicep files to json - Done' @@ -65,7 +60,7 @@ Write-Verbose 'Convert bicep files to json - Done' #region Remove Bicep metadata from json Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Processing [$($BicepFilesToConvert.count)] files" -if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] file(s) in path [$armFolderPath]", 'Modify')) { +if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] file(s) in path [$armFolderPath]", 'Set Content')) { $BicepFilesToConvert | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { function Remove-JSONMetadata { @@ -105,7 +100,7 @@ if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] file(s) in path [$a $JSONObj = $JSONFileContent | ConvertFrom-Json Remove-JSONMetadata -TemplateObject $JSONObj $JSONFileContent = $JSONObj | ConvertTo-Json -Depth 100 - Set-Content -Value $JSONFileContent -Path $JSONFilePath + Set-Content -Value $JSONFileContent -Path $JSONFilePath -WhatIf:$WhatIfPreference Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Done" } else { Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Skipped - File not found (deploy.json)" @@ -120,28 +115,31 @@ if ($CleanUp) { Write-Verbose 'Remove bicep files and folders - Remove .bicep folders' $dotBicepFolders = Get-ChildItem -Path $armFolderPath -Filter '.bicep' -Recurse -Force -Directory - $dotBicepFolders | Remove-Item -Recurse -Force -WhatIf:$WhatIfPreference + $dotBicepFolders | Remove-Item -Recurse -Force Write-Verbose 'Remove bicep files and folders - Remove .bicep folders - Done' Write-Verbose 'Remove bicep files and folders - Remove all *.bicep files' - $BicepFilesToRemove = Get-ChildItem -Path $armFolderPath -Filter '*.bicep' -Recurse -Force - $BicepFilesToRemove | Remove-Item -Force -WhatIf:$WhatIfPreference + $BicepFilesToRemove = Get-ChildItem -Path $armFolderPath -Filter '*.bicep' -Recurse -Force -File + $BicepFilesToRemove | Remove-Item -Force Write-Verbose 'Remove bicep files and folders - Remove all *.bicep files - Done' } #endregion #region Replace .bicep with .json in workflow files +Write-Verbose 'Update workflow files' + $workflowFolderPath = Join-Path -Path $rootPath -ChildPath '.github\workflows' $workflowFiles = Get-ChildItem -Path $workflowFolderPath -Filter 'ms.*.yml' -File -Force -Write-Verbose "Update workflow files - $($workflowFiles.count) files" -foreach ($workflowFile in $workflowFiles) { - Write-Verbose "$workflowFile - Processing" - $content = Get-Content -Path $workflowFile - $content = $content.Replace("deploy.bicep'", "deploy.json'") - if ($PSCmdlet.ShouldProcess("File in path [$workflowFile]", 'Overwrite')) { - Set-Content -Value $content -Path $workflowFile +Write-Verbose ('Update workflow files - Processing [{0}] files' -f $workflowFiles.count) +if ($PSCmdlet.ShouldProcess("[$($workflowFiles.count)] yml file(s) in path [$armFolderPath]", 'Set Content')) { + $workflowFiles | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { + $workflowFile = $_ + Write-Verbose "$workflowFile - Processing" + $content = Get-Content -Path $workflowFile + $content = $content.Replace("deploy.bicep'", "deploy.json'") + Set-Content -Value $content -Path $workflowFile -WhatIf:$WhatIfPreference + Write-Verbose "$workflowFile - Processing - Done" } - Write-Verbose "$workflowFile - Processing - Done" } Write-Verbose "Update workflow files - $($workflowFiles.count) files - Done" #endregion From ef5e0ce0927008f5f1f477bb4fc2e15c72461100 Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 16 Dec 2021 00:32:49 +0100 Subject: [PATCH 4/7] Updates --- utilities/tools/ConvertTo-ARMTemplate.ps1 | 26 ++++++++++++----------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/utilities/tools/ConvertTo-ARMTemplate.ps1 b/utilities/tools/ConvertTo-ARMTemplate.ps1 index cbe21faa71..b43fcb433f 100644 --- a/utilities/tools/ConvertTo-ARMTemplate.ps1 +++ b/utilities/tools/ConvertTo-ARMTemplate.ps1 @@ -50,7 +50,7 @@ Write-Verbose 'Removing existing deploy.json files - Done' #region Convert bicep files to json Write-Verbose "Convert bicep files to json - Processing [$($BicepFilesToConvert.count)] files" -if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] bicep file(s) in path [$armFolderPath]", 'Convert')) { +if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] bicep file(s) in path [$armFolderPath]", 'az bicep build')) { $BicepFilesToConvert | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { Invoke-Expression -Command "az bicep build --file '$_'" } @@ -100,27 +100,32 @@ if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] file(s) in path [$a $JSONObj = $JSONFileContent | ConvertFrom-Json Remove-JSONMetadata -TemplateObject $JSONObj $JSONFileContent = $JSONObj | ConvertTo-Json -Depth 100 - Set-Content -Value $JSONFileContent -Path $JSONFilePath -WhatIf:$WhatIfPreference + Set-Content -Value $JSONFileContent -Path $JSONFilePath Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Done" } else { Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Skipped - File not found (deploy.json)" } } } +Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Done" #endregion #region Remove bicep files and folders if ($CleanUp) { Write-Verbose 'Remove bicep files and folders' - Write-Verbose 'Remove bicep files and folders - Remove .bicep folders' $dotBicepFolders = Get-ChildItem -Path $armFolderPath -Filter '.bicep' -Recurse -Force -Directory - $dotBicepFolders | Remove-Item -Recurse -Force + Write-Verbose "Remove bicep files and folders - Remove [$($dotBicepFolders.count)] .bicep folders" + if ($PSCmdlet.ShouldProcess("[$($dotBicepFolders.count)] .bicep folder(s) in path [$armFolderPath]", 'Remove')) { + $dotBicepFolders | Remove-Item -Recurse -Force + } Write-Verbose 'Remove bicep files and folders - Remove .bicep folders - Done' - Write-Verbose 'Remove bicep files and folders - Remove all *.bicep files' $BicepFilesToRemove = Get-ChildItem -Path $armFolderPath -Filter '*.bicep' -Recurse -Force -File - $BicepFilesToRemove | Remove-Item -Force + Write-Verbose "Remove bicep files and folders - Remove [$($BicepFilesToRemove.count)] *.bicep files" + if ($PSCmdlet.ShouldProcess("[$($BicepFilesToRemove.count)] *.bicep file(s) in path [$armFolderPath]", 'Remove')) { + $BicepFilesToRemove | Remove-Item -Force + } Write-Verbose 'Remove bicep files and folders - Remove all *.bicep files - Done' } #endregion @@ -133,12 +138,9 @@ $workflowFiles = Get-ChildItem -Path $workflowFolderPath -Filter 'ms.*.yml' -Fil Write-Verbose ('Update workflow files - Processing [{0}] files' -f $workflowFiles.count) if ($PSCmdlet.ShouldProcess("[$($workflowFiles.count)] yml file(s) in path [$armFolderPath]", 'Set Content')) { $workflowFiles | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { - $workflowFile = $_ - Write-Verbose "$workflowFile - Processing" - $content = Get-Content -Path $workflowFile - $content = $content.Replace("deploy.bicep'", "deploy.json'") - Set-Content -Value $content -Path $workflowFile -WhatIf:$WhatIfPreference - Write-Verbose "$workflowFile - Processing - Done" + $content = $_ | Get-Content + $content = $content.Replace("deploy.bicep", "deploy.json") + $_ | Set-Content -Value $content } } Write-Verbose "Update workflow files - $($workflowFiles.count) files - Done" From 5e8820ce10e1e6f2f1b0ba506d7073c712eefe09 Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 16 Dec 2021 01:35:57 +0100 Subject: [PATCH 5/7] Update --- utilities/tools/ConvertTo-ARMTemplate.ps1 | 171 +++++++++++++--------- 1 file changed, 104 insertions(+), 67 deletions(-) diff --git a/utilities/tools/ConvertTo-ARMTemplate.ps1 b/utilities/tools/ConvertTo-ARMTemplate.ps1 index b43fcb433f..53e3c7f917 100644 --- a/utilities/tools/ConvertTo-ARMTemplate.ps1 +++ b/utilities/tools/ConvertTo-ARMTemplate.ps1 @@ -5,66 +5,99 @@ This script converts the module library from bicep to json based ARM templates. .DESCRIPTION The script finds all 'deploy.bicep' files and tries to convert them to json based ARM templates by using the following steps. -1. Remove existing deploy.json files (to avoid collision) -2.a Convert deploy.bicep to deploy.json using bicep build -2.b Clean up the json file, i.e removing the metadata property -3. Remove bicep files and folders -4. Replace deploy.bicep with deploy.json in workflow files +1. Remove existing deploy.json files +2. Convert bicep files to json +3. Remove Bicep metadata from json +4. Remove bicep files and folders +5. Update workflow files - Replace .bicep with .json in workflow files .PARAMETER Path -Mandatory. The path to the root of the repo. +Optional. The path to the root of the repo. -.PARAMETER CleanUp -Optional. Perform cleanup of bicep files after conversion. +.PARAMETER ConvertChildren +Optional. Convert child resource modules to bicep. + +.PARAMETER SkipMetadataCleanup +Optional. Skip Cleanup of Bicep metadata from json files + +.PARAMETER SkipBicepCleanUp +Optional. Skip removal of bicep files and folders + +.PARAMETER SkipWorkflowUpdate +Optional. Skip replacing .bicep with .json in workflow files .EXAMPLE -. .\utilities\tools\ConvertTo-ARMTemplate.ps1 -Path . -CleanUp -Verbose +. .\utilities\tools\ConvertTo-ARMTemplate.ps1 + +Converts top level bicep modules to json based ARM template, cleaning up all bicep files and folders and updating the workflow files to use the json files. + +.EXAMPLE +. .\utilities\tools\ConvertTo-ARMTemplate.ps1 -ConvertChildren -SkipMetadataCleanup -SkipBicepCleanUp -SkipWorkflowUpdate + +Only converts top level bicep modules to json based ARM template, keeping metadata in json, keeping all bicep files and folders, and not updating workflows. -Converts bicep modules to json based ARM template, cleaning up all bicep files after conversion. #> [CmdletBinding(SupportsShouldProcess)] param ( - [Parameter()] + [Parameter(Mandatory = $false)] [string] $Path = (Get-Location).Path, - [Parameter()] - [switch] $CleanUp, + [Parameter(Mandatory = $false)] + [switch] $ConvertChildren, - [Parameter()] - [switch] $KeepChildResources + [Parameter(Mandatory = $false)] + [switch] $SkipMetadataCleanup, + + [Parameter(Mandatory = $false)] + [switch] $SkipBicepCleanUp, + + [Parameter(Mandatory = $false)] + [switch] $SkipWorkflowUpdate ) $rootPath = Get-Item -Path $Path | Select-Object -ExpandProperty 'FullName' $armFolderPath = Join-Path -Path $rootPath -ChildPath 'arm' -if ($KeepChildResources) { +if ($ConvertChildren) { $BicepFilesToConvert = Get-ChildItem -Path $armFolderPath -Filter 'deploy.bicep' -Recurse -Force } else { $BicepFilesToConvert = Get-ChildItem -Path $armFolderPath -Filter 'deploy.bicep' -Recurse -Force -Depth 2 } -#region Remove existing json files -Write-Verbose 'Removing existing deploy.json files' -Get-ChildItem -Path $armFolderPath -Filter 'deploy.json' -Recurse -Force | Remove-Item -Force -Write-Verbose 'Removing existing deploy.json files - Done' +#region Remove existing deploy.json files +Write-Verbose 'Remove existing deploy.json files' + +$JsonFilesToRemove = Get-ChildItem -Path $armFolderPath -Filter 'deploy.json' -Recurse -Force -File +Write-Verbose "Remove existing deploy.json files - Remove [$($JsonFilesToRemove.count)] file(s)" +if ($PSCmdlet.ShouldProcess("[$($JsonFilesToRemove.count)] deploy.json files(s) in path [$armFolderPath]", 'Remove-Item')) { + $JsonFilesToRemove | Remove-Item -Force +} + +Write-Verbose 'Remove existing deploy.json files - Done' #endregion #region Convert bicep files to json -Write-Verbose "Convert bicep files to json - Processing [$($BicepFilesToConvert.count)] files" -if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] bicep file(s) in path [$armFolderPath]", 'az bicep build')) { +Write-Verbose 'Convert bicep files to json' + +Write-Verbose "Convert bicep files to json - Processing [$($BicepFilesToConvert.count)] file(s)" +if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] deploy.bicep file(s) in path [$armFolderPath]", 'az bicep build')) { $BicepFilesToConvert | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { Invoke-Expression -Command "az bicep build --file '$_'" } } + Write-Verbose 'Convert bicep files to json - Done' #endregion #region Remove Bicep metadata from json -Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Processing [$($BicepFilesToConvert.count)] files" -if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] file(s) in path [$armFolderPath]", 'Set Content')) { - $BicepFilesToConvert | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { +if (-not $SkipMetadataCleanup) { + Write-Verbose 'Remove Bicep metadata from json' + + Write-Verbose "Remove Bicep metadata from json - Processing [$($BicepFilesToConvert.count)] file(s)" + if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] deploy.bicep file(s) in path [$armFolderPath]", 'Set-Content')) { + $BicepFilesToConvert | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { - function Remove-JSONMetadata { - <# + function Remove-JSONMetadata { + <# .SYNOPSIS A function to recursively remove 'metadata' property from a provided object. This object is expected to be an ARM template converted to a PowerShell custom object. @@ -82,66 +115,70 @@ if ($PSCmdlet.ShouldProcess("[$($BicepFilesToConvert.count)] file(s) in path [$a #> - [CmdletBinding()] - param ( - [Parameter(Mandatory)] - [psobject] $TemplateObject - ) - $TemplateObject.PSObject.Properties.Remove('metadata') - $TemplateObject.resources | Where-Object { $_.type -eq 'Microsoft.Resources/deployments' } | ForEach-Object { - Remove-JSONMetadata -TemplateObject $_.properties.template + [CmdletBinding()] + param ( + [Parameter(Mandatory)] + [psobject] $TemplateObject + ) + $TemplateObject.PSObject.Properties.Remove('metadata') + $TemplateObject.resources | Where-Object { $_.type -eq 'Microsoft.Resources/deployments' } | ForEach-Object { + Remove-JSONMetadata -TemplateObject $_.properties.template + } } - } - $moduleFolderPath = $_.Directory.FullName - $JSONFilePath = Join-Path -Path $moduleFolderPath -ChildPath 'deploy.json' - if (Test-Path -Path $JSONFilePath) { - $JSONFileContent = Get-Content -Path $JSONFilePath - $JSONObj = $JSONFileContent | ConvertFrom-Json - Remove-JSONMetadata -TemplateObject $JSONObj - $JSONFileContent = $JSONObj | ConvertTo-Json -Depth 100 - Set-Content -Value $JSONFileContent -Path $JSONFilePath - Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Done" - } else { - Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Skipped - File not found (deploy.json)" + $moduleFolderPath = $_.Directory.FullName + $JSONFilePath = Join-Path -Path $moduleFolderPath -ChildPath 'deploy.json' + if (Test-Path -Path $JSONFilePath) { + $JSONFileContent = Get-Content -Path $JSONFilePath + $JSONObj = $JSONFileContent | ConvertFrom-Json + Remove-JSONMetadata -TemplateObject $JSONObj + $JSONFileContent = $JSONObj | ConvertTo-Json -Depth 100 + Set-Content -Value $JSONFileContent -Path $JSONFilePath + } else { + Write-Warning "Remove Bicep metadata from json - Skipped $JSONFilePath - File not found (deploy.json)" + } } } + + Write-Verbose 'Remove Bicep metadata from json - Done' } -Write-Verbose "$bicepModuleName - Remove Bicep metadata from json - Done" #endregion #region Remove bicep files and folders -if ($CleanUp) { +if (-not $SkipBicepCleanUp) { Write-Verbose 'Remove bicep files and folders' $dotBicepFolders = Get-ChildItem -Path $armFolderPath -Filter '.bicep' -Recurse -Force -Directory - Write-Verbose "Remove bicep files and folders - Remove [$($dotBicepFolders.count)] .bicep folders" - if ($PSCmdlet.ShouldProcess("[$($dotBicepFolders.count)] .bicep folder(s) in path [$armFolderPath]", 'Remove')) { + Write-Verbose "Remove bicep files and folders - Remove [$($dotBicepFolders.count)] .bicep folder(s)" + if ($PSCmdlet.ShouldProcess("[$($dotBicepFolders.count)] .bicep folder(s) in path [$armFolderPath]", 'Remove-Item')) { $dotBicepFolders | Remove-Item -Recurse -Force } - Write-Verbose 'Remove bicep files and folders - Remove .bicep folders - Done' $BicepFilesToRemove = Get-ChildItem -Path $armFolderPath -Filter '*.bicep' -Recurse -Force -File - Write-Verbose "Remove bicep files and folders - Remove [$($BicepFilesToRemove.count)] *.bicep files" - if ($PSCmdlet.ShouldProcess("[$($BicepFilesToRemove.count)] *.bicep file(s) in path [$armFolderPath]", 'Remove')) { + Write-Verbose "Remove bicep files and folders - Remove [$($BicepFilesToRemove.count)] *.bicep file(s)" + if ($PSCmdlet.ShouldProcess("[$($BicepFilesToRemove.count)] *.bicep file(s) in path [$armFolderPath]", 'Remove-Item')) { $BicepFilesToRemove | Remove-Item -Force } - Write-Verbose 'Remove bicep files and folders - Remove all *.bicep files - Done' + + Write-Verbose 'Remove bicep files and folders - Done' } #endregion -#region Replace .bicep with .json in workflow files -Write-Verbose 'Update workflow files' - -$workflowFolderPath = Join-Path -Path $rootPath -ChildPath '.github\workflows' -$workflowFiles = Get-ChildItem -Path $workflowFolderPath -Filter 'ms.*.yml' -File -Force -Write-Verbose ('Update workflow files - Processing [{0}] files' -f $workflowFiles.count) -if ($PSCmdlet.ShouldProcess("[$($workflowFiles.count)] yml file(s) in path [$armFolderPath]", 'Set Content')) { - $workflowFiles | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { - $content = $_ | Get-Content - $content = $content.Replace("deploy.bicep", "deploy.json") - $_ | Set-Content -Value $content +#region Update workflow files - Replace .bicep with .json in workflow files +if (-not $SkipWorkflowUpdate) { + Write-Verbose 'Update workflow files' + + $workflowFolderPath = Join-Path -Path $rootPath -ChildPath '.github\workflows' + $workflowFiles = Get-ChildItem -Path $workflowFolderPath -Filter 'ms.*.yml' -File -Force + Write-Verbose "Update workflow files - Processing [$($workflowFiles.count)] file(s)" + if ($PSCmdlet.ShouldProcess("[$($workflowFiles.count)] ms.*.yml file(s) in path [$armFolderPath]", 'Set-Content')) { + $workflowFiles | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { + $content = $_ | Get-Content + $content = $content.Replace('deploy.bicep', 'deploy.json') + $_ | Set-Content -Value $content + } } + + Write-Verbose 'Update workflow files - Done' } -Write-Verbose "Update workflow files - $($workflowFiles.count) files - Done" #endregion From fe584dbca7cadb1a54d6bf83d4120099e7a1a732 Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 16 Dec 2021 01:41:02 +0100 Subject: [PATCH 6/7] Update --- utilities/tools/ConvertTo-ARMTemplate.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utilities/tools/ConvertTo-ARMTemplate.ps1 b/utilities/tools/ConvertTo-ARMTemplate.ps1 index 53e3c7f917..c253fa421b 100644 --- a/utilities/tools/ConvertTo-ARMTemplate.ps1 +++ b/utilities/tools/ConvertTo-ARMTemplate.ps1 @@ -148,10 +148,10 @@ if (-not $SkipMetadataCleanup) { if (-not $SkipBicepCleanUp) { Write-Verbose 'Remove bicep files and folders' - $dotBicepFolders = Get-ChildItem -Path $armFolderPath -Filter '.bicep' -Recurse -Force -Directory - Write-Verbose "Remove bicep files and folders - Remove [$($dotBicepFolders.count)] .bicep folder(s)" - if ($PSCmdlet.ShouldProcess("[$($dotBicepFolders.count)] .bicep folder(s) in path [$armFolderPath]", 'Remove-Item')) { - $dotBicepFolders | Remove-Item -Recurse -Force + $dotBicepFoldersToRemove = Get-ChildItem -Path $armFolderPath -Filter '.bicep' -Recurse -Force -Directory + Write-Verbose "Remove bicep files and folders - Remove [$($dotBicepFoldersToRemove.count)] .bicep folder(s)" + if ($PSCmdlet.ShouldProcess("[$($dotBicepFoldersToRemove.count)] .bicep folder(s) in path [$armFolderPath]", 'Remove-Item')) { + $dotBicepFoldersToRemove | Remove-Item -Recurse -Force } $BicepFilesToRemove = Get-ChildItem -Path $armFolderPath -Filter '*.bicep' -Recurse -Force -File From eb03265fa82ffc5dbd92ae7187b9d2c52266dd08 Mon Sep 17 00:00:00 2001 From: Marius Date: Thu, 16 Dec 2021 01:42:29 +0100 Subject: [PATCH 7/7] Updates --- utilities/tools/ConvertTo-ARMTemplate.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utilities/tools/ConvertTo-ARMTemplate.ps1 b/utilities/tools/ConvertTo-ARMTemplate.ps1 index c253fa421b..769ee7fb82 100644 --- a/utilities/tools/ConvertTo-ARMTemplate.ps1 +++ b/utilities/tools/ConvertTo-ARMTemplate.ps1 @@ -169,10 +169,10 @@ if (-not $SkipWorkflowUpdate) { Write-Verbose 'Update workflow files' $workflowFolderPath = Join-Path -Path $rootPath -ChildPath '.github\workflows' - $workflowFiles = Get-ChildItem -Path $workflowFolderPath -Filter 'ms.*.yml' -File -Force - Write-Verbose "Update workflow files - Processing [$($workflowFiles.count)] file(s)" - if ($PSCmdlet.ShouldProcess("[$($workflowFiles.count)] ms.*.yml file(s) in path [$armFolderPath]", 'Set-Content')) { - $workflowFiles | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { + $workflowFilesToUpdate = Get-ChildItem -Path $workflowFolderPath -Filter 'ms.*.yml' -File -Force + Write-Verbose "Update workflow files - Processing [$($workflowFilesToUpdate.count)] file(s)" + if ($PSCmdlet.ShouldProcess("[$($workflowFilesToUpdate.count)] ms.*.yml file(s) in path [$armFolderPath]", 'Set-Content')) { + $workflowFilesToUpdate | ForEach-Object -ThrottleLimit $env:NUMBER_OF_PROCESSORS -Parallel { $content = $_ | Get-Content $content = $content.Replace('deploy.bicep', 'deploy.json') $_ | Set-Content -Value $content