diff --git a/.github/workflows/ms.datafactory.factories.yml b/.github/workflows/ms.datafactory.factories.yml index 162c0ed643..bbc3fb8aba 100644 --- a/.github/workflows/ms.datafactory.factories.yml +++ b/.github/workflows/ms.datafactory.factories.yml @@ -106,8 +106,7 @@ jobs: - name: 'Using test file [${{ matrix.moduleTestFilePaths }}]' uses: ./.github/actions/templates/validateModuleDeployment with: - templateFilePath: '${{ env.modulePath }}/deploy.bicep' - parameterFilePath: '${{ env.modulePath }}/${{ matrix.moduleTestFilePaths }}' + templateFilePath: '${{ env.modulePath }}/${{ matrix.moduleTestFilePaths }}' location: '${{ env.location }}' resourceGroupName: '${{ env.resourceGroupName }}' subscriptionId: '${{ secrets.ARM_SUBSCRIPTION_ID }}' diff --git a/modules/Microsoft.DataFactory/factories/.test/common/dependencies.bicep b/modules/Microsoft.DataFactory/factories/.test/common/dependencies.bicep new file mode 100644 index 0000000000..5317c0c39b --- /dev/null +++ b/modules/Microsoft.DataFactory/factories/.test/common/dependencies.bicep @@ -0,0 +1,133 @@ +@description('Optional. The location to deploy resources to.') +param location string = resourceGroup().location + +@description('Required. The name of the Virtual Network to create.') +param virtualNetworkName string + +@description('Required. The name of the Key Vault to create.') +param keyVaultName string + +@description('Required. The name of the Managed Identity to create.') +param managedIdentityName string + +@description('Required. The name of the Storage Account to create.') +param storageAccountName string + +resource virtualNetwork 'Microsoft.Network/virtualNetworks@2022-01-01' = { + name: virtualNetworkName + location: location + properties: { + addressSpace: { + addressPrefixes: [ + '10.0.0.0/24' + ] + } + subnets: [ + { + name: 'defaultSubnet' + properties: { + addressPrefix: '10.0.0.0/24' + } + } + ] + } +} + +resource privateDNSZone 'Microsoft.Network/privateDnsZones@2020-06-01' = { + name: 'privatelink.datafactory.azure.net' + location: 'global' + + resource virtualNetworkLinks 'virtualNetworkLinks@2020-06-01' = { + name: '${virtualNetworkName}-vnetlink' + location: 'global' + properties: { + virtualNetwork: { + id: virtualNetwork.id + } + registrationEnabled: false + } + } +} + +resource keyVault 'Microsoft.KeyVault/vaults@2022-07-01' = { + name: keyVaultName + location: location + properties: { + sku: { + family: 'A' + name: 'standard' + } + tenantId: tenant().tenantId + enablePurgeProtection: null + enabledForTemplateDeployment: true + enabledForDiskEncryption: true + enabledForDeployment: true + enableRbacAuthorization: true + accessPolicies: [] + } + + resource key 'keys@2022-07-01' = { + name: 'encryptionKey' + properties: { + kty: 'RSA' + } + } +} + +resource keyPermissions 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid('msi-${keyVault::key.id}-${location}-${managedIdentity.id}-KeyVault-Key-Read-RoleAssignment') + scope: keyVault::key + properties: { + principalId: managedIdentity.properties.principalId + // Key Vault Crypto User + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '12338af0-0e69-4776-bea7-57ae8d297424') + principalType: 'ServicePrincipal' + } +} + +resource managedIdentity 'Microsoft.ManagedIdentity/userAssignedIdentities@2018-11-30' = { + name: managedIdentityName + location: location +} + +resource storageAccount 'Microsoft.Storage/storageAccounts@2021-08-01' = { + name: storageAccountName + location: location + kind: 'StorageV2' + sku: { + name: 'Standard_LRS' + } + properties: { + allowBlobPublicAccess: false + } +} + +@description('The resource ID of the created Virtual Network Subnet.') +output subnetResourceId string = virtualNetwork.properties.subnets[0].id + +@description('The resource ID of the created Private DNS Zone.') +output privateDNSResourceId string = privateDNSZone.id + +@description('The resource ID of the created Key Vault.') +output keyVaultResourceId string = keyVault.id + +@description('The URL of the created Key Vault.') +output keyVaultUrl string = keyVault.properties.vaultUri + +@description('The principal ID of the created Managed Identity.') +output managedIdentityPrincipalId string = managedIdentity.properties.principalId + +@description('The resource ID of the created Managed Identity.') +output managedIdentityResourceId string = managedIdentity.id + +@description('The name of the created Key Vault Encryption Key.') +output keyVaultEncryptionKeyName string = keyVault::key.name + +@description('The resource ID of the created Storage Account.') +output storageAccountResourceId string = storageAccount.id + +@description('The name of the created Storage Account.') +output storageAccountName string = storageAccount.name + +@description('The Blob Endpoint of the created Storage Account.') +output storageAccountBlobEndpoint string = storageAccount.properties.primaryEndpoints.blob diff --git a/modules/Microsoft.DataFactory/factories/.test/common/deploy.test.bicep b/modules/Microsoft.DataFactory/factories/.test/common/deploy.test.bicep new file mode 100644 index 0000000000..34409149cc --- /dev/null +++ b/modules/Microsoft.DataFactory/factories/.test/common/deploy.test.bicep @@ -0,0 +1,123 @@ +targetScope = 'subscription' + +// ========== // +// Parameters // +// ========== // +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'ms.datafactory.factories-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param location string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'dffcom' + +// =========== // +// Deployments // +// =========== // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: location +} + +module resourceGroupResources 'dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-paramNested' + params: { + virtualNetworkName: 'dep-<>-vnet-${serviceShort}' + keyVaultName: 'dep-<>-kv-${serviceShort}' + managedIdentityName: 'dep-<>-msi-${serviceShort}' + storageAccountName: 'dep<>st${serviceShort}' + } +} + +// Diagnostics +// =========== +module diagnosticDependencies '../../../../.shared/dependencyConstructs/diagnostic.dependencies.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name, location)}-diagnosticDependencies' + params: { + storageAccountName: 'dep<>diasa${serviceShort}01' + logAnalyticsWorkspaceName: 'dep-<>-law-${serviceShort}' + eventHubNamespaceEventHubName: 'dep-<>-evh-${serviceShort}' + eventHubNamespaceName: 'dep-<>-evhns-${serviceShort}' + location: location + } +} + +// ============== // +// Test Execution // +// ============== // + +module testDeployment '../../deploy.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name)}-test-${serviceShort}' + params: { + name: '<>${serviceShort}001' + cMKKeyName: resourceGroupResources.outputs.keyVaultEncryptionKeyName + cMKKeyVaultResourceId: resourceGroupResources.outputs.keyVaultResourceId + cMKUserAssignedIdentityResourceId: resourceGroupResources.outputs.managedIdentityResourceId + diagnosticLogsRetentionInDays: 7 + diagnosticStorageAccountId: diagnosticDependencies.outputs.storageAccountResourceId + diagnosticWorkspaceId: diagnosticDependencies.outputs.logAnalyticsWorkspaceResourceId + diagnosticEventHubAuthorizationRuleId: diagnosticDependencies.outputs.eventHubAuthorizationRuleId + diagnosticEventHubName: diagnosticDependencies.outputs.eventHubNamespaceEventHubName + gitConfigureLater: true + integrationRuntimes: [ + { + managedVirtualNetworkName: 'default' + name: 'AutoResolveIntegrationRuntime' + type: 'Managed' + typeProperties: { + computeProperties: { + location: 'AutoResolve' + } + } + } + + { + name: 'TestRuntime' + type: 'SelfHosted' + } + ] + lock: 'CanNotDelete' + managedPrivateEndpoints: [ + { + fqdns: [ + resourceGroupResources.outputs.storageAccountBlobEndpoint + ] + groupId: 'blob' + name: '${resourceGroupResources.outputs.storageAccountName}-managed-privateEndpoint' + privateLinkResourceId: resourceGroupResources.outputs.storageAccountResourceId + } + ] + managedVirtualNetworkName: 'default' + privateEndpoints: [ + { + privateDnsZoneGroup: { + privateDNSResourceIds: [ + resourceGroupResources.outputs.privateDNSResourceId + ] + } + service: 'dataFactory' + subnetResourceId: resourceGroupResources.outputs.subnetResourceId + } + ] + roleAssignments: [ + { + principalIds: [ + resourceGroupResources.outputs.managedIdentityPrincipalId + ] + roleDefinitionIdOrName: 'Reader' + } + ] + systemAssignedIdentity: true + userAssignedIdentities: { + '${resourceGroupResources.outputs.managedIdentityResourceId}': {} + } + } +} diff --git a/modules/Microsoft.DataFactory/factories/.test/min.parameters.json b/modules/Microsoft.DataFactory/factories/.test/min.parameters.json deleted file mode 100644 index f432bf3874..0000000000 --- a/modules/Microsoft.DataFactory/factories/.test/min.parameters.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "value": "<>-adf-min-001" - } - } -} diff --git a/modules/Microsoft.DataFactory/factories/.test/min/deploy.test.bicep b/modules/Microsoft.DataFactory/factories/.test/min/deploy.test.bicep new file mode 100644 index 0000000000..25b40c4dbf --- /dev/null +++ b/modules/Microsoft.DataFactory/factories/.test/min/deploy.test.bicep @@ -0,0 +1,37 @@ +targetScope = 'subscription' + +// ========== // +// Parameters // +// ========== // +@description('Optional. The name of the resource group to deploy for testing purposes.') +@maxLength(90) +param resourceGroupName string = 'ms.datafactory.factories-${serviceShort}-rg' + +@description('Optional. The location to deploy resources to.') +param location string = deployment().location + +@description('Optional. A short identifier for the kind of deployment. Should be kept short to not run into resource-name length-constraints.') +param serviceShort string = 'dffmin' + +// =========== // +// Deployments // +// =========== // + +// General resources +// ================= +resource resourceGroup 'Microsoft.Resources/resourceGroups@2021-04-01' = { + name: resourceGroupName + location: location +} + +// ============== // +// Test Execution // +// ============== // + +module testDeployment '../../deploy.bicep' = { + scope: resourceGroup + name: '${uniqueString(deployment().name)}-test-${serviceShort}' + params: { + name: '<>${serviceShort}001' + } +} diff --git a/modules/Microsoft.DataFactory/factories/.test/parameters.json b/modules/Microsoft.DataFactory/factories/.test/parameters.json deleted file mode 100644 index add8a6a20a..0000000000 --- a/modules/Microsoft.DataFactory/factories/.test/parameters.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "value": "<>-adf-001" - }, - "lock": { - "value": "CanNotDelete" - }, - "managedVirtualNetworkName": { - "value": "default" - }, - "managedPrivateEndpoints": { - "value": [ - { - "name": "adp<>azsax001-managed-privateEndpoint", - "groupId": "blob", - "fqdns": [ - "adp<>azsax001.blob.core.windows.net" - ], - "privateLinkResourceId": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Storage/storageAccounts/adp<>azsax001" - } - ] - }, - "integrationRuntimes": { - "value": [ - { - "name": "AutoResolveIntegrationRuntime", - "type": "Managed", - "managedVirtualNetworkName": "default", - "typeProperties": { - "computeProperties": { - "location": "AutoResolve" - } - } - }, - { - "name": "TestRuntime", - "type": "SelfHosted" - } - ] - }, - "publicNetworkAccess": { - "value": "Disabled" - }, - "gitConfigureLater": { - "value": true - }, - "roleAssignments": { - "value": [ - { - "roleDefinitionIdOrName": "Reader", - "principalIds": [ - "<>" - ] - } - ] - }, - "diagnosticLogsRetentionInDays": { - "value": 7 - }, - "diagnosticStorageAccountId": { - "value": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Storage/storageAccounts/adp<>azsax001" - }, - "diagnosticWorkspaceId": { - "value": "/subscriptions/<>/resourcegroups/validation-rg/providers/microsoft.operationalinsights/workspaces/adp-<>-az-law-x-001" - }, - "diagnosticEventHubAuthorizationRuleId": { - "value": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.EventHub/namespaces/adp-<>-az-evhns-x-001/AuthorizationRules/RootManageSharedAccessKey" - }, - "diagnosticEventHubName": { - "value": "adp-<>-az-evh-x-001" - }, - "systemAssignedIdentity": { - "value": true - }, - "userAssignedIdentities": { - "value": { - "/subscriptions/<>/resourcegroups/validation-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/adp-<>-az-msi-x-001": {} - } - }, - "privateEndpoints": { - "value": [ - { - "subnetResourceId": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Network/virtualNetworks/adp-<>-az-vnet-x-001/subnets/<>-az-subnet-x-005-privateEndpoints", - "service": "dataFactory", - "privateDnsZoneGroup": { - "privateDNSResourceIds": [ - "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Network/privateDnsZones/privatelink.datafactory.azure.net" - ] - } - } - ] - }, - "cMKUserAssignedIdentityResourceId": { - "value": "/subscriptions/<>/resourcegroups/validation-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/adp-<>-az-msi-x-001" - }, - "cMKKeyName": { - "value": "keyEncryptionKey" - }, - "cMKKeyVaultResourceId": { - "value": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.KeyVault/vaults/adp-<>-az-kv-nopr-002" - } - } -} diff --git a/modules/Microsoft.DataFactory/factories/readme.md b/modules/Microsoft.DataFactory/factories/readme.md index e05a6d3c67..c01ede8a7d 100644 --- a/modules/Microsoft.DataFactory/factories/readme.md +++ b/modules/Microsoft.DataFactory/factories/readme.md @@ -25,13 +25,11 @@ ## Parameters **Required parameters** - | Parameter Name | Type | Description | | :-- | :-- | :-- | | `name` | string | The name of the Azure Factory to create. | **Optional parameters** - | Parameter Name | Type | Default Value | Allowed Values | Description | | :-- | :-- | :-- | :-- | :-- | | `cMKKeyName` | string | `''` | | The name of the customer managed key to use for encryption. | @@ -356,44 +354,7 @@ The following module usage examples are retrieved from the content of the files >**Note**: Each example lists all the required parameters first, followed by the rest - each in alphabetical order. -

Example 1: Min

- -
- -via Bicep module - -```bicep -module factories './Microsoft.DataFactory/factories/deploy.bicep' = { - name: '${uniqueString(deployment().name)}-Factories' - params: { - name: '<>-adf-min-001' - } -} -``` - -
-

- -

- -via JSON Parameter file - -```json -{ - "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "name": { - "value": "<>-adf-min-001" - } - } -} -``` - -
-

- -

Example 2: Parameters

+

Example 1: Common

@@ -401,19 +362,19 @@ module factories './Microsoft.DataFactory/factories/deploy.bicep' = { ```bicep module factories './Microsoft.DataFactory/factories/deploy.bicep' = { - name: '${uniqueString(deployment().name)}-Factories' + name: '${uniqueString(deployment().name)}-test-dffcom' params: { // Required parameters - name: '<>-adf-001' + name: '<>dffcom001' // Non-required parameters - cMKKeyName: 'keyEncryptionKey' - cMKKeyVaultResourceId: '/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.KeyVault/vaults/adp-<>-az-kv-nopr-002' - cMKUserAssignedIdentityResourceId: '/subscriptions/<>/resourcegroups/validation-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/adp-<>-az-msi-x-001' - diagnosticEventHubAuthorizationRuleId: '/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.EventHub/namespaces/adp-<>-az-evhns-x-001/AuthorizationRules/RootManageSharedAccessKey' - diagnosticEventHubName: 'adp-<>-az-evh-x-001' + cMKKeyName: '' + cMKKeyVaultResourceId: '' + cMKUserAssignedIdentityResourceId: '' + diagnosticEventHubAuthorizationRuleId: '' + diagnosticEventHubName: '' diagnosticLogsRetentionInDays: 7 - diagnosticStorageAccountId: '/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Storage/storageAccounts/adp<>azsax001' - diagnosticWorkspaceId: '/subscriptions/<>/resourcegroups/validation-rg/providers/microsoft.operationalinsights/workspaces/adp-<>-az-law-x-001' + diagnosticStorageAccountId: '' + diagnosticWorkspaceId: '' gitConfigureLater: true integrationRuntimes: [ { @@ -435,11 +396,11 @@ module factories './Microsoft.DataFactory/factories/deploy.bicep' = { managedPrivateEndpoints: [ { fqdns: [ - 'adp<>azsax001.blob.core.windows.net' + '' ] groupId: 'blob' - name: 'adp<>azsax001-managed-privateEndpoint' - privateLinkResourceId: '/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Storage/storageAccounts/adp<>azsax001' + name: '${resourceGroupResources.outputs.storageAccountName}-managed-privateEndpoint' + privateLinkResourceId: '' } ] managedVirtualNetworkName: 'default' @@ -447,25 +408,24 @@ module factories './Microsoft.DataFactory/factories/deploy.bicep' = { { privateDnsZoneGroup: { privateDNSResourceIds: [ - '/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Network/privateDnsZones/privatelink.datafactory.azure.net' + '' ] } service: 'dataFactory' - subnetResourceId: '/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Network/virtualNetworks/adp-<>-az-vnet-x-001/subnets/<>-az-subnet-x-005-privateEndpoints' + subnetResourceId: '' } ] - publicNetworkAccess: 'Disabled' roleAssignments: [ { principalIds: [ - '<>' + '' ] roleDefinitionIdOrName: 'Reader' } ] systemAssignedIdentity: true userAssignedIdentities: { - '/subscriptions/<>/resourcegroups/validation-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/adp-<>-az-msi-x-001': {} + '': {} } } } @@ -485,32 +445,32 @@ module factories './Microsoft.DataFactory/factories/deploy.bicep' = { "parameters": { // Required parameters "name": { - "value": "<>-adf-001" + "value": "<>dffcom001" }, // Non-required parameters "cMKKeyName": { - "value": "keyEncryptionKey" + "value": "" }, "cMKKeyVaultResourceId": { - "value": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.KeyVault/vaults/adp-<>-az-kv-nopr-002" + "value": "" }, "cMKUserAssignedIdentityResourceId": { - "value": "/subscriptions/<>/resourcegroups/validation-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/adp-<>-az-msi-x-001" + "value": "" }, "diagnosticEventHubAuthorizationRuleId": { - "value": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.EventHub/namespaces/adp-<>-az-evhns-x-001/AuthorizationRules/RootManageSharedAccessKey" + "value": "" }, "diagnosticEventHubName": { - "value": "adp-<>-az-evh-x-001" + "value": "" }, "diagnosticLogsRetentionInDays": { "value": 7 }, "diagnosticStorageAccountId": { - "value": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Storage/storageAccounts/adp<>azsax001" + "value": "" }, "diagnosticWorkspaceId": { - "value": "/subscriptions/<>/resourcegroups/validation-rg/providers/microsoft.operationalinsights/workspaces/adp-<>-az-law-x-001" + "value": "" }, "gitConfigureLater": { "value": true @@ -540,11 +500,11 @@ module factories './Microsoft.DataFactory/factories/deploy.bicep' = { "value": [ { "fqdns": [ - "adp<>azsax001.blob.core.windows.net" + "" ], "groupId": "blob", - "name": "adp<>azsax001-managed-privateEndpoint", - "privateLinkResourceId": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Storage/storageAccounts/adp<>azsax001" + "name": "${resourceGroupResources.outputs.storageAccountName}-managed-privateEndpoint", + "privateLinkResourceId": "" } ] }, @@ -556,22 +516,19 @@ module factories './Microsoft.DataFactory/factories/deploy.bicep' = { { "privateDnsZoneGroup": { "privateDNSResourceIds": [ - "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Network/privateDnsZones/privatelink.datafactory.azure.net" + "" ] }, "service": "dataFactory", - "subnetResourceId": "/subscriptions/<>/resourceGroups/validation-rg/providers/Microsoft.Network/virtualNetworks/adp-<>-az-vnet-x-001/subnets/<>-az-subnet-x-005-privateEndpoints" + "subnetResourceId": "" } ] }, - "publicNetworkAccess": { - "value": "Disabled" - }, "roleAssignments": { "value": [ { "principalIds": [ - "<>" + "" ], "roleDefinitionIdOrName": "Reader" } @@ -582,7 +539,7 @@ module factories './Microsoft.DataFactory/factories/deploy.bicep' = { }, "userAssignedIdentities": { "value": { - "/subscriptions/<>/resourcegroups/validation-rg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/adp-<>-az-msi-x-001": {} + "": {} } } } @@ -591,3 +548,40 @@ module factories './Microsoft.DataFactory/factories/deploy.bicep' = {

+ +

Example 2: Min

+ +
+ +via Bicep module + +```bicep +module factories './Microsoft.DataFactory/factories/deploy.bicep' = { + name: '${uniqueString(deployment().name)}-test-dffmin' + params: { + name: '<>dffmin001' + } +} +``` + +
+

+ +

+ +via JSON Parameter file + +```json +{ + "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "name": { + "value": "<>dffmin001" + } + } +} +``` + +
+