Update dependency pipeline with image template and vhd copy

.azuredevops/platformPipelines/platform.dependencies.yml

@@ -5,6 +5,10 @@ parameters:
     displayName: Enable SqlMi dependencies deployment
     type: boolean
     default: false
+  - name: deployVhdDependencies
+    displayName: Enable deployment of a vhd stored in a blob container
+    type: boolean
+    default: false
 
 trigger: none
 
@@ -219,6 +223,123 @@ stages:
               templateFilePath: $(templateFilePath)
               displayName: Default SIG and SID
 
+  - stage: deploy_imgt
+    displayName: Deploy image template
+    condition: and(succeeded(), eq('${{ parameters.deployVhdDependencies }}', true))
+    dependsOn:
+      - deploy_rolea
+      - deploy_sig
+      - deploy_sa
+    variables:
+      resourceType: 'Microsoft.VirtualMachineImages\imageTemplates'
+      saResourceType: 'Microsoft.Storage\storageAccounts'
+      templateFilePath: $(modulesPath)/$(resourceType)/deploy.bicep
+    jobs:
+      - template: /.azuredevops/pipelineTemplates/jobs.validateModuleDeployment.yml
+        parameters:
+          deploymentBlocks:
+            - path: $(dependencyPath)/$(resourceType)/parameters/parameters.json
+              templateFilePath: $(templateFilePath)
+              displayName: Image template
+              jobName: job_deploy_imgt
+      - job:
+        displayName: Trigger vhd build and store it to a storage account blob container
+        dependsOn:
+          - job_deploy_imgt
+        pool:
+          ${{ if eq(variables['vmImage'], '') }}:
+            name: $(poolName)
+          ${{ if eq(variables['poolName'], '') }}:
+            vmImage: $(vmImage)
+        variables:
+          deploymentOutput: $[ dependencies.job_deploy_imgt.outputs['DeployModule.deploymentOutput'] ]
+        steps:
+          - task: PowerShell@2
+            displayName: 'Setup agent'
+            inputs:
+              targetType: inline
+              pwsh: true
+              script: |
+                # Load used functions
+                . (Join-Path '$(System.DefaultWorkingDirectory)' 'utilities' 'pipelines' 'sharedScripts' 'Set-EnvironmentOnAgent.ps1')
+
+                # Define PS modules to install on the runner
+                $Modules = @(
+                    @{ Name = 'Az.ImageBuilder' },
+                    @{ Name = 'Az.Storage' }
+                )
+
+                # Set agent up
+                Set-EnvironmentOnAgent -PSModules $Modules
+          - task: AzurePowerShell@5
+            displayName: Trigger building new image
+            inputs:
+              azureSubscription: $(serviceConnection)
+              ScriptType: 'InlineScript'
+              Inline: |
+                # Retrieving parameters from previous job outputs
+                Write-Verbose "Retrieving parameters from previous job outputs" -Verbose
+                $imageTemplateName = (ConvertFrom-Json '$(deploymentOutput)').name
+                $imageTemplateResourceGroup = (ConvertFrom-Json '$(deploymentOutput)').resourceGroupName
+
+                # Trigger new image creation
+                Write-Verbose "Trigger new image creation with imageTemplateName $imageTemplateName and imageTemplateResourceGroup $imageTemplateResourceGroup" -Verbose
+                Start-AzImageBuilderTemplate -ImageTemplateName $imageTemplateName -ResourceGroupName $imageTemplateResourceGroup
+              azurePowerShellVersion: 'LatestVersion'
+              pwsh: true
+          - task: AzurePowerShell@5
+            displayName: Copy baked vhd to a storage account
+            inputs:
+              azureSubscription: $(serviceConnection)
+              ScriptType: 'InlineScript'
+              Inline: |
+                # Retrieving parameters from previous job outputs and parameter files
+                Write-Verbose "Retrieving parameters from previous job outputs" -Verbose
+                $imageTemplateName = (ConvertFrom-Json '$(deploymentOutput)').name
+                $imageTemplateResourceGroup = (ConvertFrom-Json '$(deploymentOutput)').resourceGroupName
+
+                Write-Verbose "Retrieving parameters from storage account parameter files" -Verbose
+                $parameterFilePath = Join-Path '$(Build.SourcesDirectory)' '$(dependencyPath)' '$(saResourceType)' 'parameters' 'parameters.json'
+                $storageAccountParameters = (ConvertFrom-Json (Get-Content -path $parameterFilePath -Raw)).parameters
+
+                Write-Verbose "Retrieving parameters from image template parameter files" -Verbose
+                $parameterFilePath = Join-Path '$(Build.SourcesDirectory)' '$(dependencyPath)' '$(resourceType)' 'parameters' 'parameters.json'
+                $imageTemplateParameters = (ConvertFrom-Json (Get-Content -path $parameterFilePath -Raw)).parameters
+
+                # Initializing parameters before the blob copy
+                Write-Verbose "Initializing source storage account parameters before the blob copy" -Verbose
+                $imgtRunOutput = Get-AzImageBuilderRunOutput -ImageTemplateName $imageTemplateName -ResourceGroupName $imageTemplateResourceGroup | Where-Object ArtifactUri -NE $null
+                $sourceUri = $imgtRunOutput.ArtifactUri
+                $sourceStorageAccountName = $sourceUri.Split('//')[1].Split('.')[0]
+                $sourceStorageAccount = Get-AzStorageAccount | Where-Object StorageAccountName -EQ $sourceStorageAccountName
+                $sourceStorageAccountContext = $sourceStorageAccount.Context
+                $sourceStorageAccountRGName = $sourceStorageAccount.ResourceGroupName
+                Write-Verbose "Retrieving artifact uri $sourceUri stored in resource group $sourceStorageAccountRGName" -Verbose
+
+                Write-Verbose "Initializing destination storage account parameters before the blob copy" -Verbose
+                $destinationStorageAccountName = $storageAccountParameters.name.value
+                $destinationStorageAccount = Get-AzStorageAccount | Where-Object StorageAccountName -EQ $destinationStorageAccountName
+                $destinationStorageAccountContext = $destinationStorageAccount.Context
+                $destinationContainerName = 'vhds'
+                $destinationBlobName = $imageTemplateParameters.name.value
+                $destinationBlobName = "$destinationBlobName.vhd"
+                Write-Verbose "Planning for destination blob name $destinationBlobName in container $destinationContainerName and storage account $destinationStorageAccountName" -Verbose
+
+                # Copying the vhd to a destination blob container
+                Write-Verbose "Copying the vhd to a destination blob container" -Verbose
+                $resourceActionInputObject = @{
+                    AbsoluteUri   = $sourceUri
+                    Context       = $sourceStorageAccountContext
+                    DestContext   = $destinationStorageAccountContext
+                    DestBlob      = $destinationBlobName
+                    DestContainer = $destinationContainerName
+                    Force         = $true
+                }
+                Start-AzStorageBlobCopy @resourceActionInputObject
+              azPSVersion: 'latest'
+              azurePowerShellVersion: 'LatestVersion'
+              pwsh: true
+
   - stage: deploy_ag
     displayName: Deploy action groups
     dependsOn:

.github/workflows/platform.dependencies.yml

@@ -8,6 +8,11 @@ on:
         description: 'Enable SqlMi dependencies deployment'
         required: false
         default: 'false'
+      deployVhdDependencies:
+        type: boolean
+        description: 'Enable deployment of a vhd stored in a blob container'
+        required: false
+        default: 'false'
 
   # push:
   #   branches:
@@ -286,6 +291,146 @@ jobs:
           managementGroupId: '${{ secrets.ARM_MGMTGROUP_ID }}'
           removeDeployment: '${{ env.removeDeployment }}'
 
+  job_deploy_imgt:
+    runs-on: ubuntu-20.04
+    name: 'Deploy image template'
+    if: github.event.inputs.deployVhdDependencies == 'true'
+    env:
+      namespace: 'Microsoft.VirtualMachineImages\imageTemplates'
+    needs:
+      - job_deploy_rolea
+      - job_deploy_sig
+    outputs:
+      imageTemplateName: ${{ steps.print_imgt_output.outputs.imageTemplateName }}
+      imageTemplateResourceGroup: ${{ steps.print_imgt_output.outputs.imageTemplateResourceGroup }}
+    strategy:
+      fail-fast: false
+      matrix:
+        parameterFilePaths: ['parameters.json']
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: 'Deploy module'
+        id: deploy_imgt
+        uses: ./.github/actions/templates/validateModuleDeployment
+        with:
+          templateFilePath: 'arm/${{ env.namespace }}/deploy.bicep'
+          parameterFilePath: '${{ env.dependencyPath }}/${{ env.namespace }}/parameters/${{ matrix.parameterFilePaths }}'
+          location: '${{ env.defaultLocation }}'
+          resourceGroupName: '${{ env.defaultResourceGroupName }}'
+          subscriptionId: '${{ secrets.ARM_SUBSCRIPTION_ID }}'
+          managementGroupId: '${{ secrets.ARM_MGMTGROUP_ID }}'
+          removeDeployment: '${{ env.removeDeployment }}'
+      - name: 'Set image template output'
+        id: print_imgt_output
+        uses: azure/powershell@v1
+        with:
+          inlineScript: |
+            $deploymentOutput = '${{ steps.deploy_imgt.outputs.deploymentOutput }}'
+            $imageTemplateName = (ConvertFrom-Json $deploymentOutput).name
+            $imageTemplateResourceGroup = (ConvertFrom-Json $deploymentOutput).resourceGroupName
+            Write-Verbose "imageTemplateName: $imageTemplateName" -Verbose
+            Write-Verbose "imageTemplateResourceGroup: $imageTemplateResourceGroup" -Verbose
+            Write-Output ('::set-output name={0}::{1}' -f 'imageTemplateName', $imageTemplateName)
+            Write-Output ('::set-output name={0}::{1}' -f 'imageTemplateResourceGroup', $imageTemplateResourceGroup)
+          azPSVersion: 'latest'
+
+  job_deploy_vhd_to_sa:
+    runs-on: ubuntu-20.04
+    name: 'Trigger vhd build and store it to a storage account blob container'
+    if: github.event.inputs.deployVhdDependencies == 'true'
+    env:
+      imgtNamespace: 'Microsoft.VirtualMachineImages\imageTemplates'
+      saNamespace: 'Microsoft.Storage\storageAccounts'
+    needs:
+      - job_deploy_imgt
+      - job_deploy_sa
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: 'Setup agent'
+        shell: pwsh
+        run: |
+          # Load used functions
+          . (Join-Path $env:GITHUB_WORKSPACE 'utilities' 'pipelines' 'sharedScripts' 'Set-EnvironmentOnAgent.ps1')
+
+          # Define PS modules to install on the runner
+          $Modules = @(
+              @{ Name = 'Az.ImageBuilder' },
+              @{ Name = 'Az.Storage' }
+          )
+
+          # Set agent up
+          Set-EnvironmentOnAgent -PSModules $Modules
+      - name: Azure Login
+        uses: azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+          enable-AzPSSession: true
+      - name: 'Trigger building new image'
+        uses: azure/powershell@v1
+        with:
+          inlineScript: |
+            Write-Verbose "Retrieving output from previous job" -Verbose
+            $imageTemplateName = '${{ needs.job_deploy_imgt.outputs.imageTemplateName }}'
+            $imageTemplateResourceGroup = '${{ needs.job_deploy_imgt.outputs.imageTemplateResourceGroup }}'
+
+            Write-Verbose "Trigger new image creation with imageTemplateName $imageTemplateName and imageTemplateResourceGroup $imageTemplateResourceGroup" -Verbose
+            Start-AzImageBuilderTemplate -ImageTemplateName $imageTemplateName -ResourceGroupName $imageTemplateResourceGroup
+          azPSVersion: 'latest'
+      - name: 'Copy baked vhd to a storage account'
+        uses: azure/powershell@v1
+        with:
+          inlineScript: |
+            # Retrieving parameters from previous job outputs and parameter files
+            Write-Verbose "Retrieving parameters from previous job outputs" -Verbose
+            $imageTemplateName = '${{ needs.job_deploy_imgt.outputs.imageTemplateName }}'
+            $imageTemplateResourceGroup = '${{ needs.job_deploy_imgt.outputs.imageTemplateResourceGroup }}'
+
+            Write-Verbose "Retrieving parameters from storage account parameter files" -Verbose
+            $parameterFilePath = Join-Path $env:GITHUB_WORKSPACE '${{ env.dependencyPath }}' '${{ env.saNamespace }}' 'parameters' 'parameters.json'
+            $storageAccountParameters = (ConvertFrom-Json (Get-Content -path $parameterFilePath -Raw)).parameters
+
+            Write-Verbose "Retrieving parameters from image template parameter files" -Verbose
+            $parameterFilePath = Join-Path $env:GITHUB_WORKSPACE '${{ env.dependencyPath }}' '${{ env.imgtNamespace }}' 'parameters' 'parameters.json'
+            $imageTemplateParameters = (ConvertFrom-Json (Get-Content -path $parameterFilePath -Raw)).parameters
+
+            # Initializing parameters before the blob copy
+            Write-Verbose "Initializing source storage account parameters before the blob copy" -Verbose
+            $imgtRunOutput = Get-AzImageBuilderRunOutput -ImageTemplateName $imageTemplateName -ResourceGroupName $imageTemplateResourceGroup | Where-Object ArtifactUri -NE $null
+            $sourceUri = $imgtRunOutput.ArtifactUri
+            $sourceStorageAccountName = $sourceUri.Split('//')[1].Split('.')[0]
+            $sourceStorageAccount = Get-AzStorageAccount | Where-Object StorageAccountName -EQ $sourceStorageAccountName
+            $sourceStorageAccountContext = $sourceStorageAccount.Context
+            $sourceStorageAccountRGName = $sourceStorageAccount.ResourceGroupName
+            Write-Verbose "Retrieving artifact uri $sourceUri stored in resource group $sourceStorageAccountRGName" -Verbose
+
+            Write-Verbose "Initializing destination storage account parameters before the blob copy" -Verbose
+            $destinationStorageAccountName = $storageAccountParameters.name.value
+            $destinationStorageAccount = Get-AzStorageAccount | Where-Object StorageAccountName -EQ $destinationStorageAccountName
+            $destinationStorageAccountContext = $destinationStorageAccount.Context
+            $destinationContainerName = 'vhds'
+            $destinationBlobName = $imageTemplateParameters.name.value
+            $destinationBlobName = "$destinationBlobName.vhd"
+            Write-Verbose "Planning for destination blob name $destinationBlobName in container $destinationContainerName and storage account $destinationStorageAccountName" -Verbose
+
+            # Copying the vhd to a destination blob container
+            Write-Verbose "Copying the vhd to a destination blob container" -Verbose
+            $resourceActionInputObject = @{
+                AbsoluteUri   = $sourceUri
+                Context       = $sourceStorageAccountContext
+                DestContext   = $destinationStorageAccountContext
+                DestBlob      = $destinationBlobName
+                DestContainer = $destinationContainerName
+                Force         = $true
+            }
+            Start-AzStorageBlobCopy @resourceActionInputObject
+          azPSVersion: 'latest'
+
   job_deploy_ag:
     runs-on: ubuntu-20.04
     name: 'Deploy action groups'

docs/wiki/TestingDesign.md

@@ -167,6 +167,7 @@ Since also dependency resources are in turn subject to dependencies with each ot
         >**Note**: This resource is deployed and configured only if sqlmi dependency resources are enabled.
       - '_adp-sxx-az-vnet-x-001_': Hosting multiple subnets to be leveraged by [virtual machine], [virtual machine scale set], [service bus], [azure NetApp files], [azure bastion], [private endpoints], [app service environment] and [application gateway] resources.
   1. AVD application group: This resource is leveraged by the [AVD workspace] resource.
+  1. Azure Image Builder template: This resource triggers the build and distribution of a VHD in a storage account. The VHD file is copied to a known storage account blob container and leveraged by [compute disks] and [compute images] resources.
 
 **Fifth level resources**: This group of resources has a dependency on one or more resources in the groups above.
 

utilities/pipelines/dependencies/Microsoft.Storage/storageAccounts/parameters/parameters.json

@@ -23,6 +23,10 @@
           {
             "name": "scripts",
             "publicAccess": "None"
+          },
+          {
+            "name": "vhds",
+            "publicAccess": "None"
           }
         ]
       }

utilities/pipelines/dependencies/Microsoft.VirtualMachineImages/imageTemplates/parameters/parameters.json

@@ -0,0 +1,44 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {
+        "name": {
+            "value": "adp-sxx-az-imgt-x-001"
+        },
+        "userMsiName": {
+            "value": "adp-sxx-az-msi-x-001"
+        },
+        "userMsiResourceGroup": {
+            "value": "validation-rg"
+        },
+        "buildTimeoutInMinutes": {
+            "value": 0
+        },
+        "vmSize": {
+            "value": "Standard_D2s_v3"
+        },
+        "osDiskSizeGB": {
+            "value": 127
+        },
+        "imageSource": {
+            "value": {
+                "type": "PlatformImage",
+                "publisher": "MicrosoftWindowsDesktop",
+                "offer": "Windows-10",
+                "sku": "19h2-evd",
+                "version": "latest"
+            }
+        },
+        "customizationSteps": {
+            "value": [
+                {
+                    "type": "WindowsRestart",
+                    "restartTimeout": "30m"
+                }
+            ]
+        },
+        "unManagedImageName": {
+            "value": "adp-sxx-az-umi-x-001"
+        }
+    }
+}