From 36fbaa1de6a2d452c6f778625a587ea6617a2518 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 18 Dec 2020 12:00:38 -0800 Subject: [PATCH 01/13] enable monitoring through policy --- .../azure-policy/azurepolicy.rules.json | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 scripts/onboarding/azure-policy/azurepolicy.rules.json diff --git a/scripts/onboarding/azure-policy/azurepolicy.rules.json b/scripts/onboarding/azure-policy/azurepolicy.rules.json new file mode 100644 index 000000000..d58bb6e04 --- /dev/null +++ b/scripts/onboarding/azure-policy/azurepolicy.rules.json @@ -0,0 +1,79 @@ +{ + "if": { + "field": "type", + "equals": "Microsoft.ContainerService/managedClusters" + }, + "then": { + "effect": "deployIfNotExists", + "details": { + "type": "Microsoft.ContainerService/managedClusters", + "name": "[field('name')]", + "roleDefinitionIds": [ + "/providers/Microsoft.Authorization/roleDefinitions/ed7f3fbd-7b88-4dd4-9017-9adb7ce333f8", + "/providers/Microsoft.Authorization/roleDefinitions/92aaf0da-9dab-42b6-94a3-d43ce8d16293", + "/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c" + ], + "existenceCondition": { + "field": "Microsoft.ContainerService/managedClusters/addonProfiles", + "contains": "logAnalyticsWorkspaceResourceID" + }, + "deployment": { + "properties": { + "mode": "incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "clusterName": { + "type": "string" + }, + "clusterResourceGroupName": { + "type": "string" + }, + "clusterLocation": { + "type": "string" + }, + "workspaceResourceId": { + "type": "string" + } + }, + "resources": [ + { + "name": "[parameters('clusterName')]", + "type": "Microsoft.ContainerService/managedClusters", + "location": "[parameters('clusterLocation')]", + "apiVersion": "2018-03-31", + "properties": { + "mode": "Incremental", + "id": "[resourceId(parameters('clusterResourceGroupName'), 'Microsoft.ContainerService/managedClusters', parameters('clusterName'))]", + "addonProfiles": { + "omsagent": { + "enabled": true, + "config": { + "logAnalyticsWorkspaceResourceID": "[parameters('workspaceResourceId')]" + } + } + } + } + } + ] + }, + "parameters": { + "clusterName": { + "value": "[field('name')]" + }, + "clusterResourceGroupName": { + "value": "[resourceGroup().name]" + }, + "clusterLocation": { + "value": "[field('location')]" + }, + "workspaceResourceId": { + "value": "" + } + } + } + } + } + } +} \ No newline at end of file From dc9aada75499174fb596b71e553d5bdf73a91288 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 18 Dec 2020 18:53:20 -0800 Subject: [PATCH 02/13] wip --- .../onboarding/azure-policy/azurepolicy.parameters.json | 9 +++++++++ scripts/onboarding/azure-policy/azurepolicy.rules.json | 7 +++---- 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 scripts/onboarding/azure-policy/azurepolicy.parameters.json diff --git a/scripts/onboarding/azure-policy/azurepolicy.parameters.json b/scripts/onboarding/azure-policy/azurepolicy.parameters.json new file mode 100644 index 000000000..2c7c1ae0b --- /dev/null +++ b/scripts/onboarding/azure-policy/azurepolicy.parameters.json @@ -0,0 +1,9 @@ +{ + "workspaceResourceId": { + "type": "string", + "metadata": { + "displayName": "Resource Id of the existing Azure Log Analytics Workspace", + "description": "Azure Monitor Log Analytics Resource ID" + } + } +} \ No newline at end of file diff --git a/scripts/onboarding/azure-policy/azurepolicy.rules.json b/scripts/onboarding/azure-policy/azurepolicy.rules.json index d58bb6e04..4121bd8ad 100644 --- a/scripts/onboarding/azure-policy/azurepolicy.rules.json +++ b/scripts/onboarding/azure-policy/azurepolicy.rules.json @@ -10,12 +10,11 @@ "name": "[field('name')]", "roleDefinitionIds": [ "/providers/Microsoft.Authorization/roleDefinitions/ed7f3fbd-7b88-4dd4-9017-9adb7ce333f8", - "/providers/Microsoft.Authorization/roleDefinitions/92aaf0da-9dab-42b6-94a3-d43ce8d16293", - "/providers/Microsoft.Authorization/roleDefinitions/b24988ac-6180-42a0-ab88-20f7382dd24c" + "/providers/Microsoft.Authorization/roleDefinitions/92aaf0da-9dab-42b6-94a3-d43ce8d16293" ], "existenceCondition": { "field": "Microsoft.ContainerService/managedClusters/addonProfiles", - "contains": "logAnalyticsWorkspaceResourceID" + "containsKey": "omsagent" }, "deployment": { "properties": { @@ -69,7 +68,7 @@ "value": "[field('location')]" }, "workspaceResourceId": { - "value": "" + "value": "[parameters('workspaceResourceId')]" } } } From fd6d614db8e572807e8f2db3c263efbcd4e04386 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 20 Dec 2020 21:45:14 -0800 Subject: [PATCH 03/13] handle tags --- scripts/onboarding/azure-policy/azurepolicy.rules.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/onboarding/azure-policy/azurepolicy.rules.json b/scripts/onboarding/azure-policy/azurepolicy.rules.json index 4121bd8ad..d342c2076 100644 --- a/scripts/onboarding/azure-policy/azurepolicy.rules.json +++ b/scripts/onboarding/azure-policy/azurepolicy.rules.json @@ -32,6 +32,9 @@ "clusterLocation": { "type": "string" }, + "clusterTags": { + "type": "object" + }, "workspaceResourceId": { "type": "string" } @@ -41,6 +44,7 @@ "name": "[parameters('clusterName')]", "type": "Microsoft.ContainerService/managedClusters", "location": "[parameters('clusterLocation')]", + "tags": "[parameters('clusterTags')]", "apiVersion": "2018-03-31", "properties": { "mode": "Incremental", @@ -67,6 +71,9 @@ "clusterLocation": { "value": "[field('location')]" }, + "clusterTags": { + "value": "[field('tags')]" + }, "workspaceResourceId": { "value": "[parameters('workspaceResourceId')]" } From bc2b6b13ddb499cac2fc7efc11e5853274042fb8 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 23 Jan 2021 19:18:28 -0800 Subject: [PATCH 04/13] wip --- .../existingClusterOnboarding.json | 61 +++++++++++++++++++ .../existingClusterParam.json | 18 ++++++ 2 files changed, 79 insertions(+) create mode 100644 scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json create mode 100644 scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json diff --git a/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json b/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json new file mode 100644 index 000000000..830298914 --- /dev/null +++ b/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json @@ -0,0 +1,61 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "arcK8sExtensionResourceId": { + "type": "string", + "metadata": { + "description": "Resource Id of the Arc K8s Extension" + } + }, + "arcK8sExtensionResourceLocation": { + "type": "string", + "metadata": { + "description": "Location of the Arc K8s Extension e.g. \"East US\"" + } + }, + "workspaceResourceId": { + "type": "string", + "metadata": { + "description": "Azure Monitor Log Analytics Resource ID" + } + }, + "workspaceDomain": { + "type": "string", + "metadata": { + "description": "Azure Monitor Log Analytics Workspace Domain e.g. opinsights.azure.com" + } + } + }, + "resources": [ + { + "name": "[split(parameters('arcK8sExtensionResourceId'),'/')[12]]", + "type": "Microsoft.KubernetesConfiguration/extensions", + "location": "[parameters('arcK8sExtensionResourceLocation')]", + "apiVersion": "2020-07-01-preview", + "properties": { + "mode": "Incremental", + "id": "[parameters('arcK8sExtensionResourceId')]", + "properties": { + "extensionType": "[split(parameters('arcK8sExtensionResourceId'),'/')[12]]", + "type": "Microsoft.KubernetesConfiguration/extensions", + "configurationSettings": { + "logAnalyticsWorkspaceResourceID": "[parameters('workspaceResourceId')]", + "omsagent.domain": "[parameters('workspaceDomain')]" + }, + "configurationProtectedSettings": { + "omsagent.secret.wsid": "[reference(parameters('workspaceResourceId'), '2015-03-20').customerId]", + "omsagent.secret.key": "[listKeys(parameters('workspaceResourceId'), '2015-03-20').primarySharedKey]" + }, + "autoUpgradeMinorVersion": true, + "releaseTrain": "Stable", + "scope": { + "Cluster": { + "releaseNamespace": "[split(parameters('arcK8sExtensionResourceId'),'/')[12]]" + } + } + } + } + } + ] +} \ No newline at end of file diff --git a/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json b/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json new file mode 100644 index 000000000..309561a9a --- /dev/null +++ b/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json @@ -0,0 +1,18 @@ +{ + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "arcK8sExtensionResourceId": { + "value": "/subscriptions/692aea0b-2d89-4e7e-ae30-fffe40782ee2/resourceGroups/gangams-akse-arc-extn-msi-euap/providers/Microsoft.Kubernetes/connectedClusters/gangams-akse-arc-extn-msi-euap/providers/Microsoft.KubernetesConfiguration/extensions/azuremonitor-containers" + }, + "arcK8sExtensionResourceLocation": { + "value": "eastus2euap" + }, + "workspaceResourceId": { + "value": "/subscriptions/692aea0b-2d89-4e7e-ae30-fffe40782ee2/resourcegroups/defaultresourcegroup-eus/providers/microsoft.operationalinsights/workspaces/defaultworkspace-692aea0b-2d89-4e7e-ae30-fffe40782ee2-eus" + }, + "workspaceDomain": { + "value": "opinsights.azure.com" + } + } +} \ No newline at end of file From 3439305e3a19ee7015c726c0748dee0bb85e4122 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 24 Jan 2021 16:18:50 -0800 Subject: [PATCH 05/13] add alias --- scripts/onboarding/azure-policy/azurepolicy.rules.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/onboarding/azure-policy/azurepolicy.rules.json b/scripts/onboarding/azure-policy/azurepolicy.rules.json index d342c2076..0f39404cd 100644 --- a/scripts/onboarding/azure-policy/azurepolicy.rules.json +++ b/scripts/onboarding/azure-policy/azurepolicy.rules.json @@ -13,8 +13,8 @@ "/providers/Microsoft.Authorization/roleDefinitions/92aaf0da-9dab-42b6-94a3-d43ce8d16293" ], "existenceCondition": { - "field": "Microsoft.ContainerService/managedClusters/addonProfiles", - "containsKey": "omsagent" + "field": "Microsoft.ContainerService/managedClusters/addonProfiles.omsagent.enabled", + "containsKey": "true" }, "deployment": { "properties": { From 73c3e4e169120134854d8bcf2a81f991175beb14 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 24 Jan 2021 19:35:18 -0800 Subject: [PATCH 06/13] wip --- scripts/onboarding/azure-policy/azurepolicy.rules.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/onboarding/azure-policy/azurepolicy.rules.json b/scripts/onboarding/azure-policy/azurepolicy.rules.json index 0f39404cd..87b00b4e4 100644 --- a/scripts/onboarding/azure-policy/azurepolicy.rules.json +++ b/scripts/onboarding/azure-policy/azurepolicy.rules.json @@ -14,7 +14,7 @@ ], "existenceCondition": { "field": "Microsoft.ContainerService/managedClusters/addonProfiles.omsagent.enabled", - "containsKey": "true" + "equals": "true" }, "deployment": { "properties": { From bf36c158d4e9f417fa7209b219be496a88c02fe6 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 25 Jan 2021 14:11:52 -0800 Subject: [PATCH 07/13] working --- .../existingClusterOnboarding.json | 66 +++++++++++-------- .../existingClusterParam.json | 8 +-- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json b/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json index 830298914..198383964 100644 --- a/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json +++ b/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json @@ -2,16 +2,16 @@ "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", "contentVersion": "1.0.0.0", "parameters": { - "arcK8sExtensionResourceId": { + "clusterResourceId": { "type": "string", "metadata": { - "description": "Resource Id of the Arc K8s Extension" + "description": "Resource Id of the Azure Arc Connected Cluster" } }, - "arcK8sExtensionResourceLocation": { + "clusterRegion": { "type": "string", "metadata": { - "description": "Location of the Arc K8s Extension e.g. \"East US\"" + "description": "Location of the zure Arc Connected Cluster e.g. \"East US\"" } }, "workspaceResourceId": { @@ -29,31 +29,45 @@ }, "resources": [ { - "name": "[split(parameters('arcK8sExtensionResourceId'),'/')[12]]", - "type": "Microsoft.KubernetesConfiguration/extensions", - "location": "[parameters('arcK8sExtensionResourceLocation')]", - "apiVersion": "2020-07-01-preview", + "type": "Microsoft.Resources/deployments", + "name": "[Concat('arc-k8s-ci-extension', '-', uniqueString(parameters('clusterResourceId')))]", + "apiVersion": "2019-05-01", + "subscriptionId": "[split(parameters('clusterResourceId'),'/')[2]]", + "resourceGroup": "[split(parameters('clusterResourceId'),'/')[4]]", "properties": { "mode": "Incremental", - "id": "[parameters('arcK8sExtensionResourceId')]", - "properties": { - "extensionType": "[split(parameters('arcK8sExtensionResourceId'),'/')[12]]", - "type": "Microsoft.KubernetesConfiguration/extensions", - "configurationSettings": { - "logAnalyticsWorkspaceResourceID": "[parameters('workspaceResourceId')]", - "omsagent.domain": "[parameters('workspaceDomain')]" - }, - "configurationProtectedSettings": { - "omsagent.secret.wsid": "[reference(parameters('workspaceResourceId'), '2015-03-20').customerId]", - "omsagent.secret.key": "[listKeys(parameters('workspaceResourceId'), '2015-03-20').primarySharedKey]" - }, - "autoUpgradeMinorVersion": true, - "releaseTrain": "Stable", - "scope": { - "Cluster": { - "releaseNamespace": "[split(parameters('arcK8sExtensionResourceId'),'/')[12]]" + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": {}, + "variables": {}, + "resources": [ + { + "type": "Microsoft.KubernetesConfiguration/extensions", + "apiVersion": "2020-07-01-preview", + "name": "azuremonitor-containers", + "location": "[parameters('clusterRegion')]", + "properties": { + "extensionType": "azuremonitor-containers", + "configurationSettings": { + "logAnalyticsWorkspaceResourceID": "[parameters('workspaceResourceId')]", + "omsagent.domain": "[parameters('workspaceDomain')]" + }, + "configurationProtectedSettings": { + "omsagent.secret.wsid": "[reference(parameters('workspaceResourceId'), '2015-03-20').customerId]", + "omsagent.secret.key": "[listKeys(parameters('workspaceResourceId'), '2015-03-20').primarySharedKey]" + }, + "autoUpgradeMinorVersion": true, + "releaseTrain": "Stable", + "scope": { + "Cluster": { + "releaseNamespace": "azuremonitor-containers" + } + } + }, + "scope": "[concat('Microsoft.Kubernetes/connectedClusters/', split(parameters('clusterResourceId'),'/')[8])]" } - } + ] } } } diff --git a/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json b/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json index 309561a9a..f6ffc56d1 100644 --- a/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json +++ b/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json @@ -2,14 +2,14 @@ "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", "contentVersion": "1.0.0.0", "parameters": { - "arcK8sExtensionResourceId": { - "value": "/subscriptions/692aea0b-2d89-4e7e-ae30-fffe40782ee2/resourceGroups/gangams-akse-arc-extn-msi-euap/providers/Microsoft.Kubernetes/connectedClusters/gangams-akse-arc-extn-msi-euap/providers/Microsoft.KubernetesConfiguration/extensions/azuremonitor-containers" + "clusterResourceId": { + "value": "/subscriptions//resourceGroups//providers/Microsoft.Kubernetes/connectedClusters/" }, - "arcK8sExtensionResourceLocation": { + "clusterRegion": { "value": "eastus2euap" }, "workspaceResourceId": { - "value": "/subscriptions/692aea0b-2d89-4e7e-ae30-fffe40782ee2/resourcegroups/defaultresourcegroup-eus/providers/microsoft.operationalinsights/workspaces/defaultworkspace-692aea0b-2d89-4e7e-ae30-fffe40782ee2-eus" + "value": "/subscriptions//resourcegroups//providers/microsoft.operationalinsights/workspaces/" }, "workspaceDomain": { "value": "opinsights.azure.com" From 73b984decd495524114600f41d7a86fbc3afa715 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 25 Jan 2021 15:21:58 -0800 Subject: [PATCH 08/13] updates --- .../existingClusterOnboarding.json | 45 ++++++++++++++++++- .../existingClusterParam.json | 5 ++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json b/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json index 198383964..ce25533c7 100644 --- a/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json +++ b/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json @@ -11,7 +11,7 @@ "clusterRegion": { "type": "string", "metadata": { - "description": "Location of the zure Arc Connected Cluster e.g. \"East US\"" + "description": "Location of the Azure Arc Connected Cluster Resource e.g. \"eastus\"" } }, "workspaceResourceId": { @@ -20,6 +20,12 @@ "description": "Azure Monitor Log Analytics Resource ID" } }, + "workspaceRegion": { + "type": "string", + "metadata": { + "description": "Azure Monitor Log Analytics Workspace region e.g. \"eastus\"" + } + }, "workspaceDomain": { "type": "string", "metadata": { @@ -28,12 +34,49 @@ } }, "resources": [ + { + "type": "Microsoft.Resources/deployments", + "name": "[Concat('ContainerInsights', '-', uniqueString(parameters('workspaceResourceId')))]", + "apiVersion": "2017-05-10", + "subscriptionId": "[split(parameters('workspaceResourceId'),'/')[2]]", + "resourceGroup": "[split(parameters('workspaceResourceId'),'/')[4]]", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": {}, + "variables": {}, + "resources": [ + { + "apiVersion": "2015-11-01-preview", + "type": "Microsoft.OperationsManagement/solutions", + "location": "[parameters('workspaceRegion')]", + "name": "[Concat('ContainerInsights', '(', split(parameters('workspaceResourceId'),'/')[8], ')')]", + "properties": { + "workspaceResourceId": "[parameters('workspaceResourceId')]" + }, + "plan": { + "name": "[Concat('ContainerInsights', '(', split(parameters('workspaceResourceId'),'/')[8], ')')]", + "product": "[Concat('OMSGallery/', 'ContainerInsights')]", + "promotionCode": "", + "publisher": "Microsoft" + } + } + ] + }, + "parameters": {} + } + }, { "type": "Microsoft.Resources/deployments", "name": "[Concat('arc-k8s-ci-extension', '-', uniqueString(parameters('clusterResourceId')))]", "apiVersion": "2019-05-01", "subscriptionId": "[split(parameters('clusterResourceId'),'/')[2]]", "resourceGroup": "[split(parameters('clusterResourceId'),'/')[4]]", + "dependsOn": [ + "[Concat('ContainerInsights', '-', uniqueString(parameters('workspaceResourceId')))]" + ], "properties": { "mode": "Incremental", "template": { diff --git a/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json b/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json index f6ffc56d1..4d2bcc05d 100644 --- a/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json +++ b/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json @@ -6,11 +6,14 @@ "value": "/subscriptions//resourceGroups//providers/Microsoft.Kubernetes/connectedClusters/" }, "clusterRegion": { - "value": "eastus2euap" + "value": "" }, "workspaceResourceId": { "value": "/subscriptions//resourcegroups//providers/microsoft.operationalinsights/workspaces/" }, + "workspaceRegion": { + "value": "" + }, "workspaceDomain": { "value": "opinsights.azure.com" } From 73ad91f597ea1423fd1554863900728c547caa2f Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 25 Jan 2021 15:39:47 -0800 Subject: [PATCH 09/13] working --- .../templates/arc-k8s-extension/existingClusterParam.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json b/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json index 4d2bcc05d..4bb5fdc80 100644 --- a/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json +++ b/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json @@ -15,7 +15,7 @@ "value": "" }, "workspaceDomain": { - "value": "opinsights.azure.com" + "value": "" } } } \ No newline at end of file From 7f30b42d649b859a84c3ed21c762c95832ab95b2 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 25 Jan 2021 22:20:36 -0800 Subject: [PATCH 10/13] with deployment name --- .../azure-policy/azurepolicy.rules.json | 42 +++++-- .../existingClusterOnboarding.json | 118 ------------------ .../existingClusterParam.json | 21 ---- 3 files changed, 29 insertions(+), 152 deletions(-) delete mode 100644 scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json delete mode 100644 scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json diff --git a/scripts/onboarding/azure-policy/azurepolicy.rules.json b/scripts/onboarding/azure-policy/azurepolicy.rules.json index 87b00b4e4..1b6e20ced 100644 --- a/scripts/onboarding/azure-policy/azurepolicy.rules.json +++ b/scripts/onboarding/azure-policy/azurepolicy.rules.json @@ -34,28 +34,44 @@ }, "clusterTags": { "type": "object" - }, + }, "workspaceResourceId": { "type": "string" } }, "resources": [ { - "name": "[parameters('clusterName')]", - "type": "Microsoft.ContainerService/managedClusters", - "location": "[parameters('clusterLocation')]", - "tags": "[parameters('clusterTags')]", - "apiVersion": "2018-03-31", + "type": "Microsoft.Resources/deployments", + "name": "[Concat('aks-monitoring-policy', '-', uniqueString(parameters('clusterName')))]", + "apiVersion": "2019-05-01", "properties": { "mode": "Incremental", - "id": "[resourceId(parameters('clusterResourceGroupName'), 'Microsoft.ContainerService/managedClusters', parameters('clusterName'))]", - "addonProfiles": { - "omsagent": { - "enabled": true, - "config": { - "logAnalyticsWorkspaceResourceID": "[parameters('workspaceResourceId')]" + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[parameters('clusterName')]", + "type": "Microsoft.ContainerService/managedClusters", + "location": "[parameters('clusterLocation')]", + "tags": "[parameters('clusterTags')]", + "apiVersion": "2018-03-31", + "properties": { + "mode": "Incremental", + "id": "[resourceId(parameters('clusterResourceGroupName'), 'Microsoft.ContainerService/managedClusters', parameters('clusterName'))]", + "addonProfiles": { + "omsagent": { + "enabled": true, + "config": { + "logAnalyticsWorkspaceResourceID": "[parameters('workspaceResourceId')]" + } + } + } + } } - } + ] } } } diff --git a/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json b/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json deleted file mode 100644 index ce25533c7..000000000 --- a/scripts/onboarding/templates/arc-k8s-extension/existingClusterOnboarding.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "clusterResourceId": { - "type": "string", - "metadata": { - "description": "Resource Id of the Azure Arc Connected Cluster" - } - }, - "clusterRegion": { - "type": "string", - "metadata": { - "description": "Location of the Azure Arc Connected Cluster Resource e.g. \"eastus\"" - } - }, - "workspaceResourceId": { - "type": "string", - "metadata": { - "description": "Azure Monitor Log Analytics Resource ID" - } - }, - "workspaceRegion": { - "type": "string", - "metadata": { - "description": "Azure Monitor Log Analytics Workspace region e.g. \"eastus\"" - } - }, - "workspaceDomain": { - "type": "string", - "metadata": { - "description": "Azure Monitor Log Analytics Workspace Domain e.g. opinsights.azure.com" - } - } - }, - "resources": [ - { - "type": "Microsoft.Resources/deployments", - "name": "[Concat('ContainerInsights', '-', uniqueString(parameters('workspaceResourceId')))]", - "apiVersion": "2017-05-10", - "subscriptionId": "[split(parameters('workspaceResourceId'),'/')[2]]", - "resourceGroup": "[split(parameters('workspaceResourceId'),'/')[4]]", - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "variables": {}, - "resources": [ - { - "apiVersion": "2015-11-01-preview", - "type": "Microsoft.OperationsManagement/solutions", - "location": "[parameters('workspaceRegion')]", - "name": "[Concat('ContainerInsights', '(', split(parameters('workspaceResourceId'),'/')[8], ')')]", - "properties": { - "workspaceResourceId": "[parameters('workspaceResourceId')]" - }, - "plan": { - "name": "[Concat('ContainerInsights', '(', split(parameters('workspaceResourceId'),'/')[8], ')')]", - "product": "[Concat('OMSGallery/', 'ContainerInsights')]", - "promotionCode": "", - "publisher": "Microsoft" - } - } - ] - }, - "parameters": {} - } - }, - { - "type": "Microsoft.Resources/deployments", - "name": "[Concat('arc-k8s-ci-extension', '-', uniqueString(parameters('clusterResourceId')))]", - "apiVersion": "2019-05-01", - "subscriptionId": "[split(parameters('clusterResourceId'),'/')[2]]", - "resourceGroup": "[split(parameters('clusterResourceId'),'/')[4]]", - "dependsOn": [ - "[Concat('ContainerInsights', '-', uniqueString(parameters('workspaceResourceId')))]" - ], - "properties": { - "mode": "Incremental", - "template": { - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", - "contentVersion": "1.0.0.0", - "parameters": {}, - "variables": {}, - "resources": [ - { - "type": "Microsoft.KubernetesConfiguration/extensions", - "apiVersion": "2020-07-01-preview", - "name": "azuremonitor-containers", - "location": "[parameters('clusterRegion')]", - "properties": { - "extensionType": "azuremonitor-containers", - "configurationSettings": { - "logAnalyticsWorkspaceResourceID": "[parameters('workspaceResourceId')]", - "omsagent.domain": "[parameters('workspaceDomain')]" - }, - "configurationProtectedSettings": { - "omsagent.secret.wsid": "[reference(parameters('workspaceResourceId'), '2015-03-20').customerId]", - "omsagent.secret.key": "[listKeys(parameters('workspaceResourceId'), '2015-03-20').primarySharedKey]" - }, - "autoUpgradeMinorVersion": true, - "releaseTrain": "Stable", - "scope": { - "Cluster": { - "releaseNamespace": "azuremonitor-containers" - } - } - }, - "scope": "[concat('Microsoft.Kubernetes/connectedClusters/', split(parameters('clusterResourceId'),'/')[8])]" - } - ] - } - } - } - ] -} \ No newline at end of file diff --git a/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json b/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json deleted file mode 100644 index 4bb5fdc80..000000000 --- a/scripts/onboarding/templates/arc-k8s-extension/existingClusterParam.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", - "contentVersion": "1.0.0.0", - "parameters": { - "clusterResourceId": { - "value": "/subscriptions//resourceGroups//providers/Microsoft.Kubernetes/connectedClusters/" - }, - "clusterRegion": { - "value": "" - }, - "workspaceResourceId": { - "value": "/subscriptions//resourcegroups//providers/microsoft.operationalinsights/workspaces/" - }, - "workspaceRegion": { - "value": "" - }, - "workspaceDomain": { - "value": "" - } - } -} \ No newline at end of file From f7518f0550d70313b2d35c4549d9ebc2d6fd4a3a Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 26 Jan 2021 16:07:15 -0800 Subject: [PATCH 11/13] doc updates --- .../azure-policy.json | 113 ++++++++++++++++++ .../azurepolicy.parameters.json | 9 ++ .../azurepolicy.rules.json | 2 +- .../azure-policy/azurepolicy.parameters.json | 9 -- .../enable-monitoring-using-policy.md | 64 ++++++++++ 5 files changed, 187 insertions(+), 10 deletions(-) create mode 100644 scripts/onboarding/aks/onboarding-using-azure-policy/azure-policy.json create mode 100644 scripts/onboarding/aks/onboarding-using-azure-policy/azurepolicy.parameters.json rename scripts/onboarding/{azure-policy => aks/onboarding-using-azure-policy}/azurepolicy.rules.json (99%) delete mode 100644 scripts/onboarding/azure-policy/azurepolicy.parameters.json create mode 100644 scripts/onboarding/enable-monitoring-using-policy.md diff --git a/scripts/onboarding/aks/onboarding-using-azure-policy/azure-policy.json b/scripts/onboarding/aks/onboarding-using-azure-policy/azure-policy.json new file mode 100644 index 000000000..c68bfed17 --- /dev/null +++ b/scripts/onboarding/aks/onboarding-using-azure-policy/azure-policy.json @@ -0,0 +1,113 @@ +{ + "mode": "Indexed", + "policyRule": { + "if": { + "field": "type", + "equals": "Microsoft.ContainerService/managedClusters" + }, + "then": { + "effect": "deployIfNotExists", + "details": { + "type": "Microsoft.ContainerService/managedClusters", + "name": "[field('name')]", + "roleDefinitionIds": [ + "/providers/Microsoft.Authorization/roleDefinitions/ed7f3fbd-7b88-4dd4-9017-9adb7ce333f8", + "/providers/Microsoft.Authorization/roleDefinitions/92aaf0da-9dab-42b6-94a3-d43ce8d16293" + ], + "existenceCondition": { + "field": "Microsoft.ContainerService/managedClusters/addonProfiles.omsagent.enabled", + "equals": "true" + }, + "deployment": { + "properties": { + "mode": "incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "clusterName": { + "type": "string" + }, + "clusterResourceGroupName": { + "type": "string" + }, + "clusterLocation": { + "type": "string" + }, + "clusterTags": { + "type": "object" + }, + "workspaceResourceId": { + "type": "string" + } + }, + "resources": [ + { + "type": "Microsoft.Resources/deployments", + "name": "[Concat('aks-monitoring-policy', '-', uniqueString(parameters('clusterName')))]", + "apiVersion": "2019-05-01", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": {}, + "variables": {}, + "resources": [ + { + "name": "[parameters('clusterName')]", + "type": "Microsoft.ContainerService/managedClusters", + "location": "[parameters('clusterLocation')]", + "tags": "[parameters('clusterTags')]", + "apiVersion": "2018-03-31", + "properties": { + "mode": "Incremental", + "id": "[resourceId(parameters('clusterResourceGroupName'), 'Microsoft.ContainerService/managedClusters', parameters('clusterName'))]", + "addonProfiles": { + "omsagent": { + "enabled": true, + "config": { + "logAnalyticsWorkspaceResourceID": "[parameters('workspaceResourceId')]" + } + } + } + } + } + ] + } + } + } + ] + }, + "parameters": { + "clusterName": { + "value": "[field('name')]" + }, + "clusterResourceGroupName": { + "value": "[resourceGroup().name]" + }, + "clusterLocation": { + "value": "[field('location')]" + }, + "clusterTags": { + "value": "[field('tags')]" + }, + "workspaceResourceId": { + "value": "[parameters('workspaceResourceId')]" + } + } + } + } + } + } + }, + "parameters": { + "workspaceResourceId": { + "type": "String", + "metadata": { + "displayName": "Resource Id of the existing Azure Log Analytics Workspace", + "description": "Azure Monitor Log Analytics Resource ID" + } + } + } +} diff --git a/scripts/onboarding/aks/onboarding-using-azure-policy/azurepolicy.parameters.json b/scripts/onboarding/aks/onboarding-using-azure-policy/azurepolicy.parameters.json new file mode 100644 index 000000000..6281cdade --- /dev/null +++ b/scripts/onboarding/aks/onboarding-using-azure-policy/azurepolicy.parameters.json @@ -0,0 +1,9 @@ +{ + "workspaceResourceId": { + "type": "string", + "metadata": { + "displayName": "Resource Id of the existing Azure Log Analytics Workspace", + "description": "Azure Monitor Log Analytics Resource ID" + } + } +} diff --git a/scripts/onboarding/azure-policy/azurepolicy.rules.json b/scripts/onboarding/aks/onboarding-using-azure-policy/azurepolicy.rules.json similarity index 99% rename from scripts/onboarding/azure-policy/azurepolicy.rules.json rename to scripts/onboarding/aks/onboarding-using-azure-policy/azurepolicy.rules.json index 1b6e20ced..a113441ce 100644 --- a/scripts/onboarding/azure-policy/azurepolicy.rules.json +++ b/scripts/onboarding/aks/onboarding-using-azure-policy/azurepolicy.rules.json @@ -98,4 +98,4 @@ } } } -} \ No newline at end of file +} diff --git a/scripts/onboarding/azure-policy/azurepolicy.parameters.json b/scripts/onboarding/azure-policy/azurepolicy.parameters.json deleted file mode 100644 index 2c7c1ae0b..000000000 --- a/scripts/onboarding/azure-policy/azurepolicy.parameters.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "workspaceResourceId": { - "type": "string", - "metadata": { - "displayName": "Resource Id of the existing Azure Log Analytics Workspace", - "description": "Azure Monitor Log Analytics Resource ID" - } - } -} \ No newline at end of file diff --git a/scripts/onboarding/enable-monitoring-using-policy.md b/scripts/onboarding/enable-monitoring-using-policy.md new file mode 100644 index 000000000..4ea271521 --- /dev/null +++ b/scripts/onboarding/enable-monitoring-using-policy.md @@ -0,0 +1,64 @@ +# How to enable AKS Monitoring Addon via Azure Policy +This doc describes how to enable AKS Monitoring Addon using Azure Custom Policy.Monitoring Addon Custom Policy can be assigned +either at subscription or resource group scope. If Azure Log Analytics workspace and AKS cluster are in different subscriptions then Managed Identity used by Policy assignnment has to have required role permissions on both the subscriptions or least on the resource of the Azure Log Aalytics workspace. +Similarly, If the policy scoped to Resource Group, then Managed Identity should have required role permissions on the Log Analytics workspace if the workspace not in the selected Resource Group scope. + +Monitoring Addon require following roles on the Managed Identity used by Azure Policy + - [azure-kubernetes-service-contributor-role](https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#azure-kubernetes-service-contributor-role) + - [log-analytics-contributor](https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#log-analytics-contributor) + +## Create and Assign Policy definition using Azure Portal + +### Create Policy Definition + +1. Download the Azure Custom Policy definition to enable AKS Monitoring Addon +``` sh + curl -o azurepolicy.json -L https://aka.ms/aks-enable-monitoring-custom-policy +``` +2. Navigate to https://portal.azure.com/#blade/Microsoft_Azure_Policy/PolicyMenuBlade/Definitions and create policy definition with the following details in the Policy definition create dialogue box + + - Pick any Azure Subscription where you want to store Policy Definition + - Name - '(Preview)AKS-Monitoring-Addon' + - Description - 'Azure Custom Policy to enable Monitoring Addon onto Azure Kubernetes Cluster(s) in specified scope' + - Category - Choose "use existing" and pick 'Kubernetes' from drop down + - Remove the existing sample rules and copy the contents of azurepolicy.json downloaded in step #1 above + +### Assign Policy Definition to Specified Scope + +> Note: Managed Identity will be created automatically and assigned specified roles in the Policy definition. + +3. Navigate to https://portal.azure.com/#blade/Microsoft_Azure_Policy/PolicyMenuBlade/Definitions and select the Policy Definition 'AKS Monitoring Addon' +4. Click an Assignment and select Scope, Exclusions (if any) +5. Provide the Resource Id of the Azure Log Analytics Workspace. The Resource Id should be in this format `/subscriptions//resourceGroups//providers/Microsoft.OperationalInsights/workspaces/` +6. Create Remediation task in case if you want apply to policy to existing AKS clusters in selected scope +7. Click and Review & Create Option to create Policy Assignment + +## Create and Assign Policy definition using Azure CLI + +### Create Policy Definition + +1. Download the Azure Custom Policy definition rules and parameters files + ``` sh + curl -o azurepolicy.rules.json -L https://aka.ms/aks-enable-monitoring-custom-policy-rules + curl -o azurepolicy.parameters.json -L https://aka.ms/aks-enable-monitoring-custom-policy-parameters + ``` +2. Create policy definition using below command + + ``` sh + az login # login to azure cloud + az account set -s + az policy definition create --name "(Preview)AKS-Monitoring-Addon" --display-name "(Preview)AKS-Monitoring-Addon" --mode Indexed --metadata version=1.0.0 category=Kubernetes --rules azurepolicy.rules.json --params azurepolicy.parameters.json + ``` +### Assign Policy Definition to Specified Scope + +3. Create policy assignment + +``` sh +az policy assignment create --name aks-monitoring --policy "(Preview)AKS-Monitoring-Addon" --assign-identity --identity-scope /subscriptions/ --role Contributor --scope /subscriptions/ --location --role Contributor --scope /subscriptions/692aea0b-2d89-4e7e-ae30-fffe40782ee2 -p "{ \"workspaceResourceId\": { \"value\": \"/subscriptions//resourcegroups//providers/microsoft.operationalinsights/workspaces/\" } }" + +``` + +## References +- https://docs.microsoft.com/en-us/azure/governance/policy/ +- https://docs.microsoft.com/en-us/azure/governance/policy/how-to/remediate-resources#how-remediation-security-works +- https://docs.microsoft.com/en-us/cli/azure/install-azure-cli \ No newline at end of file From bdc5ccb9e2bdb7efd0bc5350e1e75b0a12a041a8 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 26 Jan 2021 16:14:00 -0800 Subject: [PATCH 12/13] doc updates --- scripts/onboarding/enable-monitoring-using-policy.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/onboarding/enable-monitoring-using-policy.md b/scripts/onboarding/enable-monitoring-using-policy.md index 4ea271521..e21adab04 100644 --- a/scripts/onboarding/enable-monitoring-using-policy.md +++ b/scripts/onboarding/enable-monitoring-using-policy.md @@ -1,7 +1,6 @@ # How to enable AKS Monitoring Addon via Azure Policy This doc describes how to enable AKS Monitoring Addon using Azure Custom Policy.Monitoring Addon Custom Policy can be assigned -either at subscription or resource group scope. If Azure Log Analytics workspace and AKS cluster are in different subscriptions then Managed Identity used by Policy assignnment has to have required role permissions on both the subscriptions or least on the resource of the Azure Log Aalytics workspace. -Similarly, If the policy scoped to Resource Group, then Managed Identity should have required role permissions on the Log Analytics workspace if the workspace not in the selected Resource Group scope. +either at subscription or resource group scope. If Azure Log Analytics workspace and AKS cluster are in different subscriptions then Managed Identity used by Policy assignnment has to have required role permissions on both the subscriptions or least on the resource of the Azure Log Aalytics workspace. Similarly, If the policy scoped to Resource Group, then Managed Identity should have required role permissions on the Log Analytics workspace if the workspace not in the selected Resource Group scope. Monitoring Addon require following roles on the Managed Identity used by Azure Policy - [azure-kubernetes-service-contributor-role](https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles#azure-kubernetes-service-contributor-role) @@ -45,7 +44,8 @@ Monitoring Addon require following roles on the Managed Identity used by Azure P 2. Create policy definition using below command ``` sh - az login # login to azure cloud + az cloud set -n # set the Azure cloud + az login # login to cloud environment az account set -s az policy definition create --name "(Preview)AKS-Monitoring-Addon" --display-name "(Preview)AKS-Monitoring-Addon" --mode Indexed --metadata version=1.0.0 category=Kubernetes --rules azurepolicy.rules.json --params azurepolicy.parameters.json ``` @@ -54,11 +54,11 @@ Monitoring Addon require following roles on the Managed Identity used by Azure P 3. Create policy assignment ``` sh -az policy assignment create --name aks-monitoring --policy "(Preview)AKS-Monitoring-Addon" --assign-identity --identity-scope /subscriptions/ --role Contributor --scope /subscriptions/ --location --role Contributor --scope /subscriptions/692aea0b-2d89-4e7e-ae30-fffe40782ee2 -p "{ \"workspaceResourceId\": { \"value\": \"/subscriptions//resourcegroups//providers/microsoft.operationalinsights/workspaces/\" } }" - +az policy assignment create --name aks-monitoring-addon --policy "(Preview)AKS-Monitoring-Addon" --assign-identity --identity-scope /subscriptions/ --role Contributor --scope /subscriptions/ --location --role Contributor --scope /subscriptions/692aea0b-2d89-4e7e-ae30-fffe40782ee2 -p "{ \"workspaceResourceId\": { \"value\": \"/subscriptions//resourcegroups//providers/microsoft.operationalinsights/workspaces/\" } }" ``` ## References - https://docs.microsoft.com/en-us/azure/governance/policy/ - https://docs.microsoft.com/en-us/azure/governance/policy/how-to/remediate-resources#how-remediation-security-works -- https://docs.microsoft.com/en-us/cli/azure/install-azure-cli \ No newline at end of file +- https://docs.microsoft.com/en-us/cli/azure/install-azure-cli +- https://docs.microsoft.com/en-us/azure/azure-monitor/insights/container-insights-overview \ No newline at end of file From 370e5eefa63872e84a94ecf9d97362f6df13e4f9 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 29 Jan 2021 16:06:12 -0800 Subject: [PATCH 13/13] fix typo in the docs --- scripts/onboarding/enable-monitoring-using-policy.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/onboarding/enable-monitoring-using-policy.md b/scripts/onboarding/enable-monitoring-using-policy.md index e21adab04..e1e395ecc 100644 --- a/scripts/onboarding/enable-monitoring-using-policy.md +++ b/scripts/onboarding/enable-monitoring-using-policy.md @@ -54,7 +54,7 @@ Monitoring Addon require following roles on the Managed Identity used by Azure P 3. Create policy assignment ``` sh -az policy assignment create --name aks-monitoring-addon --policy "(Preview)AKS-Monitoring-Addon" --assign-identity --identity-scope /subscriptions/ --role Contributor --scope /subscriptions/ --location --role Contributor --scope /subscriptions/692aea0b-2d89-4e7e-ae30-fffe40782ee2 -p "{ \"workspaceResourceId\": { \"value\": \"/subscriptions//resourcegroups//providers/microsoft.operationalinsights/workspaces/\" } }" +az policy assignment create --name aks-monitoring-addon --policy "(Preview)AKS-Monitoring-Addon" --assign-identity --identity-scope /subscriptions/ --role Contributor --scope /subscriptions/ --location --role Contributor --scope /subscriptions/ -p "{ \"workspaceResourceId\": { \"value\": \"/subscriptions//resourcegroups//providers/microsoft.operationalinsights/workspaces/\" } }" ``` ## References