From ef6ba1d9065d15964d70092894754bdfc14aa3c2 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 18 Jan 2021 19:17:35 -0800 Subject: [PATCH 01/15] wip --- charts/azuremonitor-containers/Chart.yaml | 2 +- .../templates/omsagent-arc-k8s-crd.yaml | 12 ++++++++++++ charts/azuremonitor-containers/values.yaml | 9 ++++++--- source/plugins/ruby/arc_k8s_cluster_identity.rb | 3 +++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/charts/azuremonitor-containers/Chart.yaml b/charts/azuremonitor-containers/Chart.yaml index a809a4e69..ce64fd1ce 100644 --- a/charts/azuremonitor-containers/Chart.yaml +++ b/charts/azuremonitor-containers/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: 7.0.0-1 description: Helm chart for deploying Azure Monitor container monitoring agent in Kubernetes name: azuremonitor-containers -version: 2.8.0 +version: 2.8.1 kubeVersion: "^1.10.0-0" keywords: - monitoring diff --git a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml index ebdd5ea3f..ef597815b 100644 --- a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml @@ -1,4 +1,15 @@ {{- if or ( contains "microsoft.kubernetes/connectedclusters" (.Values.Azure.Cluster.ResourceId | lower) ) ( contains "microsoft.kubernetes/connectedclusters" (.Values.omsagent.env.clusterId | lower)) }} +apiVersion: clusterconfig.azure.com/v1beta1 +kind: AzureExtensionIdentity +metadata: + name: {{ .Values.Azure.Extension.ResourceName }} + namespace: azure-arc +spec: + serviceAccounts: + - name: omsagent + namespace: kube-system + tokenNamespace: azure-arc +--- apiVersion: clusterconfig.azure.com/v1beta1 kind: AzureClusterIdentityRequest metadata: @@ -6,4 +17,5 @@ metadata: namespace: azure-arc spec: audience: https://monitoring.azure.com/ + resourceId: {{ .Values.Azure.Extension.ResourceName }} {{- end }} diff --git a/charts/azuremonitor-containers/values.yaml b/charts/azuremonitor-containers/values.yaml index debd66b0b..d874d9baa 100644 --- a/charts/azuremonitor-containers/values.yaml +++ b/charts/azuremonitor-containers/values.yaml @@ -4,15 +4,18 @@ ## Microsoft OMS Agent image for kubernetes cluster monitoring ## ref: https://github.com/microsoft/Docker-Provider/tree/ci_prod -## Values of ResourceId and Region under Azure->Cluster being populated by Azure Arc K8s RP during the installation of the extension +## Values of under Azure are being populated by Azure Arc K8s RP during the installation of the extension Azure: Cluster: Region: ResourceId: + Extension: + ResourceName: azuremonitor-containers + IdentityType: "SystemAssigned" omsagent: image: - repo: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod" - tag: "ciprod01112021" + repo: "mcr.microsoft.com/azuremonitor/containerinsights/cidev" + tag: "ciarcext01182021" tagWindows: "win-ciprod01112021" pullPolicy: IfNotPresent dockerProviderVersion: "12.0.0-0" diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index ef55c3257..e40dbca05 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -19,6 +19,7 @@ class ArcK8sClusterIdentity @@secret_resource_uri_template = "%{kube_api_server_url}/api/v1/namespaces/%{cluster_identity_token_secret_namespace}/secrets/%{token_secret_name}" @@azure_monitor_custom_metrics_audience = "https://monitoring.azure.com/" @@cluster_identity_request_kind = "AzureClusterIdentityRequest" + @@container_insights_extension_name = "azuremonitor-containers" def initialize @LogPath = "/var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log" @@ -34,6 +35,7 @@ def initialize end @http_client = get_http_client @service_account_token = get_service_account_token + @extensionResourceId = ENV["AKS_RESOURCE_ID"] + "/extensions/" + @@container_insights_extension_name @log.info "initialize complete @ #{Time.now.utc.iso8601}" end @@ -211,6 +213,7 @@ def get_crd_request_body body["metadata"]["namespace"] = @@cluster_identity_resource_namespace body["spec"] = {} body["spec"]["audience"] = @@azure_monitor_custom_metrics_audience + body["spec"]["resourceId"] = @extensionResourceId return body end end From 206bf40eed8e135409d9efc0b9e206717da5c4bf Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 18 Jan 2021 21:46:23 -0800 Subject: [PATCH 02/15] add env var for the arc k8s extension name --- .../templates/omsagent-daemonset.yaml | 4 +++- .../templates/omsagent-deployment.yaml | 4 +++- source/plugins/ruby/arc_k8s_cluster_identity.rb | 8 ++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml index 3d29ede42..d5dadc9dc 100644 --- a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml @@ -70,8 +70,10 @@ spec: valueFrom: fieldRef: fieldPath: status.hostIP + - name: ARC_K8S_EXTENSION_NAME + value: {{ .Values.Azure.Extension.ResourceName | quote }} - name: USER_ASSIGNED_IDENTITY_CLIENT_ID - value: "" + value: "" {{- if .Values.omsagent.logsettings.logflushintervalsecs }} - name: FBIT_SERVICE_FLUSH_INTERVAL value: {{ .Values.omsagent.logsettings.logflushintervalsecs | quote }} diff --git a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml index 8609d25c9..d7451492e 100644 --- a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml @@ -67,8 +67,10 @@ spec: valueFrom: fieldRef: fieldPath: status.hostIP + - name: ARC_K8S_EXTENSION_NAME + value: {{ .Values.Azure.Extension.ResourceName | quote }} - name: USER_ASSIGNED_IDENTITY_CLIENT_ID - value: "" + value: "" securityContext: privileged: true ports: diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index e40dbca05..d0d9131bf 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -18,8 +18,7 @@ class ArcK8sClusterIdentity @@crd_resource_uri_template = "%{kube_api_server_url}/apis/%{cluster_config_crd_api_version}/namespaces/%{cluster_identity_resource_namespace}/azureclusteridentityrequests/%{cluster_identity_resource_name}" @@secret_resource_uri_template = "%{kube_api_server_url}/api/v1/namespaces/%{cluster_identity_token_secret_namespace}/secrets/%{token_secret_name}" @@azure_monitor_custom_metrics_audience = "https://monitoring.azure.com/" - @@cluster_identity_request_kind = "AzureClusterIdentityRequest" - @@container_insights_extension_name = "azuremonitor-containers" + @@cluster_identity_request_kind = "AzureClusterIdentityRequest" def initialize @LogPath = "/var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log" @@ -34,8 +33,9 @@ def initialize @log.warn "got api server url nil from KubernetesApiClient.getKubeAPIServerUrl @ #{Time.now.utc.iso8601}" end @http_client = get_http_client - @service_account_token = get_service_account_token - @extensionResourceId = ENV["AKS_RESOURCE_ID"] + "/extensions/" + @@container_insights_extension_name + @service_account_token = get_service_account_token + @extensionResourceId = ENV["AKS_RESOURCE_ID"] + "/extensions/" + ENV["ARC_K8S_EXTENSION_NAME"] + @log.info "extension resource id: #{@extensionResourceId} @ #{Time.now.utc.iso8601}" @log.info "initialize complete @ #{Time.now.utc.iso8601}" end From 898fbff8dad67e84f1f1c62bd7674aab2f34c046 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 18 Jan 2021 23:09:42 -0800 Subject: [PATCH 03/15] chart update --- charts/azuremonitor-containers/Chart.yaml | 2 +- charts/azuremonitor-containers/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/azuremonitor-containers/Chart.yaml b/charts/azuremonitor-containers/Chart.yaml index ce64fd1ce..9c8014ed0 100644 --- a/charts/azuremonitor-containers/Chart.yaml +++ b/charts/azuremonitor-containers/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: 7.0.0-1 description: Helm chart for deploying Azure Monitor container monitoring agent in Kubernetes name: azuremonitor-containers -version: 2.8.1 +version: 2.8.2 kubeVersion: "^1.10.0-0" keywords: - monitoring diff --git a/charts/azuremonitor-containers/values.yaml b/charts/azuremonitor-containers/values.yaml index d874d9baa..0379b573f 100644 --- a/charts/azuremonitor-containers/values.yaml +++ b/charts/azuremonitor-containers/values.yaml @@ -15,7 +15,7 @@ Azure: omsagent: image: repo: "mcr.microsoft.com/azuremonitor/containerinsights/cidev" - tag: "ciarcext01182021" + tag: "ciarcext01182021-1" tagWindows: "win-ciprod01112021" pullPolicy: IfNotPresent dockerProviderVersion: "12.0.0-0" From 949cfedbf5db830987e7ec1c343da1e5eb97a589 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 20 Jan 2021 11:03:10 -0800 Subject: [PATCH 04/15] extension msi updates --- charts/azuremonitor-containers/Chart.yaml | 2 +- .../templates/omsagent-daemonset.yaml | 6 ++++-- .../templates/omsagent-deployment.yaml | 6 ++++-- .../templates/omsagent-rbac.yaml | 4 ---- source/plugins/ruby/arc_k8s_cluster_identity.rb | 12 +++++++++--- 5 files changed, 18 insertions(+), 12 deletions(-) diff --git a/charts/azuremonitor-containers/Chart.yaml b/charts/azuremonitor-containers/Chart.yaml index 9c8014ed0..00f3f49ed 100644 --- a/charts/azuremonitor-containers/Chart.yaml +++ b/charts/azuremonitor-containers/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: 7.0.0-1 description: Helm chart for deploying Azure Monitor container monitoring agent in Kubernetes name: azuremonitor-containers -version: 2.8.2 +version: 2.8.3 kubeVersion: "^1.10.0-0" keywords: - monitoring diff --git a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml index d5dadc9dc..c406453c2 100644 --- a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml @@ -70,8 +70,10 @@ spec: valueFrom: fieldRef: fieldPath: status.hostIP - - name: ARC_K8S_EXTENSION_NAME - value: {{ .Values.Azure.Extension.ResourceName | quote }} + - name: ARC_K8S_EXTENSION_RESOURCE_NAME + value: {{ .Values.Azure.Extension.ResourceName | quote }} + - name: ARC_K8S_EXTENSION_IDENTITY_TYPE + value: {{ .Values.Azure.Extension.IdentityType | quote }} - name: USER_ASSIGNED_IDENTITY_CLIENT_ID value: "" {{- if .Values.omsagent.logsettings.logflushintervalsecs }} diff --git a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml index d7451492e..54e0a3907 100644 --- a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml @@ -67,8 +67,10 @@ spec: valueFrom: fieldRef: fieldPath: status.hostIP - - name: ARC_K8S_EXTENSION_NAME - value: {{ .Values.Azure.Extension.ResourceName | quote }} + - name: ARC_K8S_EXTENSION_RESOURCE_NAME + value: {{ .Values.Azure.Extension.ResourceName | quote }} + - name: ARC_K8S_EXTENSION_IDENTITY_TYPE + value: {{ .Values.Azure.Extension.IdentityType | quote }} - name: USER_ASSIGNED_IDENTITY_CLIENT_ID value: "" securityContext: diff --git a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml index bd4e9baf3..cc3bd9a02 100644 --- a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml @@ -33,10 +33,6 @@ rules: verbs: ["get", "create", "patch"] - nonResourceURLs: ["/metrics"] verbs: ["get"] -- apiGroups: [""] - resources: ["secrets"] - resourceNames: ["container-insights-clusteridentityrequest-token"] - verbs: ["get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index d0d9131bf..ee08bec46 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -34,8 +34,9 @@ def initialize end @http_client = get_http_client @service_account_token = get_service_account_token - @extensionResourceId = ENV["AKS_RESOURCE_ID"] + "/extensions/" + ENV["ARC_K8S_EXTENSION_NAME"] - @log.info "extension resource id: #{@extensionResourceId} @ #{Time.now.utc.iso8601}" + @extensionResourceName = ENV["ARC_K8S_EXTENSION_RESOURCE_NAME"] + @extensionIdentityType = ENV["ARC_K8S_EXTENSION_IDENTITY_TYPE"] + @log.info "extension resource name: #{@extensionResourceName}, extension identity type:#{@extensionIdentityType} @ #{Time.now.utc.iso8601}" @log.info "initialize complete @ #{Time.now.utc.iso8601}" end @@ -213,7 +214,12 @@ def get_crd_request_body body["metadata"]["namespace"] = @@cluster_identity_resource_namespace body["spec"] = {} body["spec"]["audience"] = @@azure_monitor_custom_metrics_audience - body["spec"]["resourceId"] = @extensionResourceId + if !@extensionIdentityType.nil? && !@extensionIdentityType.empty? && @extensionIdentityType.downcase == "systemassigned" + if @extensionResourceName.nil? || @extensionResourceName.empty? + @log.warn "Got Extension resource name either nil or empty" + else + body["spec"]["resourceId"] = @extensionResourceName + end return body end end From fe8dfb190aad76b1a324f6c007c6b380ee101c75 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 20 Jan 2021 11:27:58 -0800 Subject: [PATCH 05/15] fix bug --- source/plugins/ruby/arc_k8s_cluster_identity.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index ee08bec46..ba4fffa43 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -215,11 +215,12 @@ def get_crd_request_body body["spec"] = {} body["spec"]["audience"] = @@azure_monitor_custom_metrics_audience if !@extensionIdentityType.nil? && !@extensionIdentityType.empty? && @extensionIdentityType.downcase == "systemassigned" - if @extensionResourceName.nil? || @extensionResourceName.empty? - @log.warn "Got Extension resource name either nil or empty" - else + if !@extensionResourceName.nil? && !@extensionResourceName.empty? body["spec"]["resourceId"] = @extensionResourceName + else + @log.warn "Got Extension resource name either nil or empty" end + end return body end end From 2b09a3df55ae4c950c9d988e45472a6471a59d87 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 20 Jan 2021 13:26:32 -0800 Subject: [PATCH 06/15] revert chart and image to prod version --- charts/azuremonitor-containers/Chart.yaml | 2 +- charts/azuremonitor-containers/values.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/azuremonitor-containers/Chart.yaml b/charts/azuremonitor-containers/Chart.yaml index 00f3f49ed..a809a4e69 100644 --- a/charts/azuremonitor-containers/Chart.yaml +++ b/charts/azuremonitor-containers/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: 7.0.0-1 description: Helm chart for deploying Azure Monitor container monitoring agent in Kubernetes name: azuremonitor-containers -version: 2.8.3 +version: 2.8.0 kubeVersion: "^1.10.0-0" keywords: - monitoring diff --git a/charts/azuremonitor-containers/values.yaml b/charts/azuremonitor-containers/values.yaml index 0379b573f..63a6645a6 100644 --- a/charts/azuremonitor-containers/values.yaml +++ b/charts/azuremonitor-containers/values.yaml @@ -15,7 +15,7 @@ Azure: omsagent: image: repo: "mcr.microsoft.com/azuremonitor/containerinsights/cidev" - tag: "ciarcext01182021-1" + tag: "ciprod01112021" tagWindows: "win-ciprod01112021" pullPolicy: IfNotPresent dockerProviderVersion: "12.0.0-0" From 9ce998afa55545f609f8dd47b0cc7b1fd024126b Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 20 Jan 2021 13:28:50 -0800 Subject: [PATCH 07/15] minor text changes --- source/plugins/ruby/arc_k8s_cluster_identity.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index ba4fffa43..a83979af7 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -36,7 +36,7 @@ def initialize @service_account_token = get_service_account_token @extensionResourceName = ENV["ARC_K8S_EXTENSION_RESOURCE_NAME"] @extensionIdentityType = ENV["ARC_K8S_EXTENSION_IDENTITY_TYPE"] - @log.info "extension resource name: #{@extensionResourceName}, extension identity type:#{@extensionIdentityType} @ #{Time.now.utc.iso8601}" + @log.info "extension resource name:#{@extensionResourceName}, extension identity type:#{@extensionIdentityType} @ #{Time.now.utc.iso8601}" @log.info "initialize complete @ #{Time.now.utc.iso8601}" end @@ -151,7 +151,7 @@ def renew_near_expiry_token() update_response = @http_client.request(update_request) @log.info "Got response of #{update_response.code} for PATCH #{crd_request_uri} @ #{Time.now.utc.iso8601}" if update_response.code.to_i == 404 - @log.info "since crd resource doesnt exist since creating crd resource : #{@@cluster_identity_resource_name} @ #{Time.now.utc.iso8601}" + @log.info "since crd resource doesnt exist hence creating crd resource : #{@@cluster_identity_resource_name} @ #{Time.now.utc.iso8601}" create_request = Net::HTTP::Post.new(crd_request_uri) create_request["Content-Type"] = "application/json" create_request["Authorization"] = "Bearer #{@service_account_token}" From bdda28327bdd64171f0ea614992c5fa20234d382 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 20 Jan 2021 13:32:08 -0800 Subject: [PATCH 08/15] image tag to prod --- charts/azuremonitor-containers/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/azuremonitor-containers/values.yaml b/charts/azuremonitor-containers/values.yaml index 63a6645a6..884f48395 100644 --- a/charts/azuremonitor-containers/values.yaml +++ b/charts/azuremonitor-containers/values.yaml @@ -14,7 +14,7 @@ Azure: IdentityType: "SystemAssigned" omsagent: image: - repo: "mcr.microsoft.com/azuremonitor/containerinsights/cidev" + repo: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod" tag: "ciprod01112021" tagWindows: "win-ciprod01112021" pullPolicy: IfNotPresent From b6f650f7c16874771b5c04645bf6738c215c48e5 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 20 Jan 2021 16:47:25 -0800 Subject: [PATCH 09/15] wip --- charts/azuremonitor-containers/templates/omsagent-rbac.yaml | 4 ++++ charts/azuremonitor-containers/values.yaml | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml index cc3bd9a02..bd4e9baf3 100644 --- a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml @@ -33,6 +33,10 @@ rules: verbs: ["get", "create", "patch"] - nonResourceURLs: ["/metrics"] verbs: ["get"] +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["container-insights-clusteridentityrequest-token"] + verbs: ["get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/charts/azuremonitor-containers/values.yaml b/charts/azuremonitor-containers/values.yaml index 884f48395..45e7c509a 100644 --- a/charts/azuremonitor-containers/values.yaml +++ b/charts/azuremonitor-containers/values.yaml @@ -10,8 +10,8 @@ Azure: Region: ResourceId: Extension: - ResourceName: azuremonitor-containers - IdentityType: "SystemAssigned" + ResourceName: + IdentityType: omsagent: image: repo: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod" From 2534b135ef905a6ab247c1670d3478ae1c698d83 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 20 Jan 2021 16:54:53 -0800 Subject: [PATCH 10/15] wip --- .../azuremonitor-containers/templates/omsagent-daemonset.yaml | 2 +- .../azuremonitor-containers/templates/omsagent-deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml index c406453c2..0164f7aa0 100644 --- a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml @@ -75,7 +75,7 @@ spec: - name: ARC_K8S_EXTENSION_IDENTITY_TYPE value: {{ .Values.Azure.Extension.IdentityType | quote }} - name: USER_ASSIGNED_IDENTITY_CLIENT_ID - value: "" + value: "" {{- if .Values.omsagent.logsettings.logflushintervalsecs }} - name: FBIT_SERVICE_FLUSH_INTERVAL value: {{ .Values.omsagent.logsettings.logflushintervalsecs | quote }} diff --git a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml index 54e0a3907..7cf2462eb 100644 --- a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml @@ -72,7 +72,7 @@ spec: - name: ARC_K8S_EXTENSION_IDENTITY_TYPE value: {{ .Values.Azure.Extension.IdentityType | quote }} - name: USER_ASSIGNED_IDENTITY_CLIENT_ID - value: "" + value: "" securityContext: privileged: true ports: From 7d3386690fbeb2d0a725e6c86d64d75781f22ccb Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 21 Jan 2021 09:15:15 -0800 Subject: [PATCH 11/15] wip --- charts/azuremonitor-containers/templates/omsagent-rbac.yaml | 4 ---- charts/azuremonitor-containers/values.yaml | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml index bd4e9baf3..cc3bd9a02 100644 --- a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml @@ -33,10 +33,6 @@ rules: verbs: ["get", "create", "patch"] - nonResourceURLs: ["/metrics"] verbs: ["get"] -- apiGroups: [""] - resources: ["secrets"] - resourceNames: ["container-insights-clusteridentityrequest-token"] - verbs: ["get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/charts/azuremonitor-containers/values.yaml b/charts/azuremonitor-containers/values.yaml index 45e7c509a..d77d4146b 100644 --- a/charts/azuremonitor-containers/values.yaml +++ b/charts/azuremonitor-containers/values.yaml @@ -10,8 +10,8 @@ Azure: Region: ResourceId: Extension: - ResourceName: - IdentityType: + ResourceName: "azuremonitor-containers" + IdentityType: "SystemAssigned" omsagent: image: repo: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod" From 62f3cc6ad8224d3f9d7ed223af2ace8e183bd957 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 21 Jan 2021 19:06:39 -0800 Subject: [PATCH 12/15] wip --- charts/azuremonitor-containers/Chart.yaml | 2 +- .../templates/omsagent-arc-k8s-crd.yaml | 15 +++++++++++++-- .../templates/omsagent-daemonset.yaml | 8 ++++---- .../templates/omsagent-deployment.yaml | 8 ++++---- .../templates/omsagent-rbac.yaml | 7 +++++++ charts/azuremonitor-containers/values.yaml | 9 +++------ source/plugins/ruby/arc_k8s_cluster_identity.rb | 13 ++++--------- 7 files changed, 36 insertions(+), 26 deletions(-) diff --git a/charts/azuremonitor-containers/Chart.yaml b/charts/azuremonitor-containers/Chart.yaml index a809a4e69..139c54885 100644 --- a/charts/azuremonitor-containers/Chart.yaml +++ b/charts/azuremonitor-containers/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: 7.0.0-1 description: Helm chart for deploying Azure Monitor container monitoring agent in Kubernetes name: azuremonitor-containers -version: 2.8.0 +version: 2.8.6 kubeVersion: "^1.10.0-0" keywords: - monitoring diff --git a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml index ef597815b..d0bdf69a8 100644 --- a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml @@ -1,8 +1,10 @@ {{- if or ( contains "microsoft.kubernetes/connectedclusters" (.Values.Azure.Cluster.ResourceId | lower) ) ( contains "microsoft.kubernetes/connectedclusters" (.Values.omsagent.env.clusterId | lower)) }} +#extension model +{{- if not (empty .Values.Azure.Extension.Name) }} apiVersion: clusterconfig.azure.com/v1beta1 kind: AzureExtensionIdentity metadata: - name: {{ .Values.Azure.Extension.ResourceName }} + name: {{ .Values.Azure.Extension.Name }} namespace: azure-arc spec: serviceAccounts: @@ -17,5 +19,14 @@ metadata: namespace: azure-arc spec: audience: https://monitoring.azure.com/ - resourceId: {{ .Values.Azure.Extension.ResourceName }} + resourceId: {{ .Values.Azure.Extension.Name }} +{{- else }} +apiVersion: clusterconfig.azure.com/v1beta1 +kind: AzureClusterIdentityRequest +metadata: + name: container-insights-clusteridentityrequest + namespace: azure-arc +spec: + audience: https://monitoring.azure.com/ {{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml index 0164f7aa0..ebf80357f 100644 --- a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml @@ -70,10 +70,10 @@ spec: valueFrom: fieldRef: fieldPath: status.hostIP - - name: ARC_K8S_EXTENSION_RESOURCE_NAME - value: {{ .Values.Azure.Extension.ResourceName | quote }} - - name: ARC_K8S_EXTENSION_IDENTITY_TYPE - value: {{ .Values.Azure.Extension.IdentityType | quote }} + {{- if not (empty .Values.Azure.Extension.Name) }} + - name: ARC_K8S_EXTENSION_NAME + value: {{ .Values.Azure.Extension.Name | quote }} + {{- end }} - name: USER_ASSIGNED_IDENTITY_CLIENT_ID value: "" {{- if .Values.omsagent.logsettings.logflushintervalsecs }} diff --git a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml index 7cf2462eb..4e1017411 100644 --- a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml @@ -67,10 +67,10 @@ spec: valueFrom: fieldRef: fieldPath: status.hostIP - - name: ARC_K8S_EXTENSION_RESOURCE_NAME - value: {{ .Values.Azure.Extension.ResourceName | quote }} - - name: ARC_K8S_EXTENSION_IDENTITY_TYPE - value: {{ .Values.Azure.Extension.IdentityType | quote }} + {{- if not (empty .Values.Azure.Extension.Name) }} + - name: ARC_K8S_EXTENSION_NAME + value: {{ .Values.Azure.Extension.Name | quote }} + {{- end }} - name: USER_ASSIGNED_IDENTITY_CLIENT_ID value: "" securityContext: diff --git a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml index cc3bd9a02..58fb26e09 100644 --- a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml @@ -33,6 +33,13 @@ rules: verbs: ["get", "create", "patch"] - nonResourceURLs: ["/metrics"] verbs: ["get"] +#extension model grants access as part of the extension msi hence not required +{{- if (empty .Values.Azure.Extension.Name) }} +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["container-insights-clusteridentityrequest-token"] + verbs: ["get"] +{{- end }} --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/charts/azuremonitor-containers/values.yaml b/charts/azuremonitor-containers/values.yaml index d77d4146b..bbd507d4d 100644 --- a/charts/azuremonitor-containers/values.yaml +++ b/charts/azuremonitor-containers/values.yaml @@ -8,14 +8,11 @@ Azure: Cluster: Region: - ResourceId: - Extension: - ResourceName: "azuremonitor-containers" - IdentityType: "SystemAssigned" + ResourceId: omsagent: image: - repo: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod" - tag: "ciprod01112021" + repo: "mcr.microsoft.com/azuremonitor/containerinsights/cidev" + tag: "ciarcext01182021-3" tagWindows: "win-ciprod01112021" pullPolicy: IfNotPresent dockerProviderVersion: "12.0.0-0" diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index a83979af7..ec44f0937 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -34,9 +34,8 @@ def initialize end @http_client = get_http_client @service_account_token = get_service_account_token - @extensionResourceName = ENV["ARC_K8S_EXTENSION_RESOURCE_NAME"] - @extensionIdentityType = ENV["ARC_K8S_EXTENSION_IDENTITY_TYPE"] - @log.info "extension resource name:#{@extensionResourceName}, extension identity type:#{@extensionIdentityType} @ #{Time.now.utc.iso8601}" + @extensionName = ENV["ARC_K8S_EXTENSION_NAME"] + @log.info "extension name:#{@extensionName} @ #{Time.now.utc.iso8601}" @log.info "initialize complete @ #{Time.now.utc.iso8601}" end @@ -214,12 +213,8 @@ def get_crd_request_body body["metadata"]["namespace"] = @@cluster_identity_resource_namespace body["spec"] = {} body["spec"]["audience"] = @@azure_monitor_custom_metrics_audience - if !@extensionIdentityType.nil? && !@extensionIdentityType.empty? && @extensionIdentityType.downcase == "systemassigned" - if !@extensionResourceName.nil? && !@extensionResourceName.empty? - body["spec"]["resourceId"] = @extensionResourceName - else - @log.warn "Got Extension resource name either nil or empty" - end + if !@extensionName.nil? && !@extensionName.empty? + body["spec"]["resourceId"] = @extensionName end return body end From 8ec9591856663249cc7b631456bcbd398781bacc Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 21 Jan 2021 22:55:12 -0800 Subject: [PATCH 13/15] final updates --- charts/azuremonitor-containers/Chart.yaml | 2 +- .../azuremonitor-containers/templates/omsagent-rbac.yaml | 3 ++- charts/azuremonitor-containers/values.yaml | 9 ++++++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/charts/azuremonitor-containers/Chart.yaml b/charts/azuremonitor-containers/Chart.yaml index 139c54885..a809a4e69 100644 --- a/charts/azuremonitor-containers/Chart.yaml +++ b/charts/azuremonitor-containers/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v1 appVersion: 7.0.0-1 description: Helm chart for deploying Azure Monitor container monitoring agent in Kubernetes name: azuremonitor-containers -version: 2.8.6 +version: 2.8.0 kubeVersion: "^1.10.0-0" keywords: - monitoring diff --git a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml index 58fb26e09..5a10d6c71 100644 --- a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml @@ -33,7 +33,8 @@ rules: verbs: ["get", "create", "patch"] - nonResourceURLs: ["/metrics"] verbs: ["get"] -#extension model grants access as part of the extension msi hence not required +#arc k8s extension model grants access as part of the extension msi +#remove this once the extension available public preview {{- if (empty .Values.Azure.Extension.Name) }} - apiGroups: [""] resources: ["secrets"] diff --git a/charts/azuremonitor-containers/values.yaml b/charts/azuremonitor-containers/values.yaml index bbd507d4d..341b9fb65 100644 --- a/charts/azuremonitor-containers/values.yaml +++ b/charts/azuremonitor-containers/values.yaml @@ -8,11 +8,14 @@ Azure: Cluster: Region: - ResourceId: + ResourceId: + Extension: + Name: "" + ResourceId: "" omsagent: image: - repo: "mcr.microsoft.com/azuremonitor/containerinsights/cidev" - tag: "ciarcext01182021-3" + repo: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod" + tag: "ciprod01112021" tagWindows: "win-ciprod01112021" pullPolicy: IfNotPresent dockerProviderVersion: "12.0.0-0" From 3ea7fc340eac3e8608e37a3095903e02b1d7f8b0 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 21 Jan 2021 23:00:05 -0800 Subject: [PATCH 14/15] fix whitespaces --- .../azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml | 2 +- .../azuremonitor-containers/templates/omsagent-daemonset.yaml | 2 +- .../azuremonitor-containers/templates/omsagent-deployment.yaml | 2 +- source/plugins/ruby/arc_k8s_cluster_identity.rb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml index d0bdf69a8..ae193db04 100644 --- a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml @@ -29,4 +29,4 @@ metadata: spec: audience: https://monitoring.azure.com/ {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml index ebf80357f..595edd7bb 100644 --- a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml @@ -75,7 +75,7 @@ spec: value: {{ .Values.Azure.Extension.Name | quote }} {{- end }} - name: USER_ASSIGNED_IDENTITY_CLIENT_ID - value: "" + value: "" {{- if .Values.omsagent.logsettings.logflushintervalsecs }} - name: FBIT_SERVICE_FLUSH_INTERVAL value: {{ .Values.omsagent.logsettings.logflushintervalsecs | quote }} diff --git a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml index 4e1017411..ecd0b705b 100644 --- a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml @@ -72,7 +72,7 @@ spec: value: {{ .Values.Azure.Extension.Name | quote }} {{- end }} - name: USER_ASSIGNED_IDENTITY_CLIENT_ID - value: "" + value: "" securityContext: privileged: true ports: diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index ec44f0937..7824f3d4e 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -18,7 +18,7 @@ class ArcK8sClusterIdentity @@crd_resource_uri_template = "%{kube_api_server_url}/apis/%{cluster_config_crd_api_version}/namespaces/%{cluster_identity_resource_namespace}/azureclusteridentityrequests/%{cluster_identity_resource_name}" @@secret_resource_uri_template = "%{kube_api_server_url}/api/v1/namespaces/%{cluster_identity_token_secret_namespace}/secrets/%{token_secret_name}" @@azure_monitor_custom_metrics_audience = "https://monitoring.azure.com/" - @@cluster_identity_request_kind = "AzureClusterIdentityRequest" + @@cluster_identity_request_kind = "AzureClusterIdentityRequest" def initialize @LogPath = "/var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log" From 8eae81f92849df718ddc4b955ac91b61a70e0192 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 22 Jan 2021 16:10:09 -0800 Subject: [PATCH 15/15] simplify crd yaml --- .../templates/omsagent-arc-k8s-crd.yaml | 12 +++--------- .../templates/omsagent-rbac.yaml | 2 +- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml index ae193db04..b7482b8b5 100644 --- a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml @@ -12,6 +12,7 @@ spec: namespace: kube-system tokenNamespace: azure-arc --- +{{- end }} apiVersion: clusterconfig.azure.com/v1beta1 kind: AzureClusterIdentityRequest metadata: @@ -19,14 +20,7 @@ metadata: namespace: azure-arc spec: audience: https://monitoring.azure.com/ + {{- if not (empty .Values.Azure.Extension.Name) }} resourceId: {{ .Values.Azure.Extension.Name }} -{{- else }} -apiVersion: clusterconfig.azure.com/v1beta1 -kind: AzureClusterIdentityRequest -metadata: - name: container-insights-clusteridentityrequest - namespace: azure-arc -spec: - audience: https://monitoring.azure.com/ -{{- end }} + {{- end }} {{- end }} diff --git a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml index 5a10d6c71..5db5c2dab 100644 --- a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml @@ -34,7 +34,7 @@ rules: - nonResourceURLs: ["/metrics"] verbs: ["get"] #arc k8s extension model grants access as part of the extension msi -#remove this once the extension available public preview +#remove this explicit permission once the extension available in public preview {{- if (empty .Values.Azure.Extension.Name) }} - apiGroups: [""] resources: ["secrets"]