From 2ad718197003db2f097bf5a4592ad3c92faa3e16 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 28 Jul 2020 20:06:25 -0700 Subject: [PATCH 01/24] cluster identity token --- .../installer/datafiles/base_container.data | 2 +- .../ruby/arc_k8s_cluster_identity_token.rb | 106 ++++++++++++++++++ 2 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 source/plugins/ruby/arc_k8s_cluster_identity_token.rb diff --git a/build/linux/installer/datafiles/base_container.data b/build/linux/installer/datafiles/base_container.data index f07e71b2d..960678936 100644 --- a/build/linux/installer/datafiles/base_container.data +++ b/build/linux/installer/datafiles/base_container.data @@ -50,7 +50,7 @@ MAINTAINER: 'Microsoft Corporation' /opt/microsoft/omsagent/plugin/kubernetes_container_inventory.rb; source/plugins/ruby/kubernetes_container_inventory.rb; 644; root; root /opt/microsoft/omsagent/plugin/proxy_utils.rb; source/plugins/ruby/proxy_utils.rb; 644; root; root - +/opt/microsoft/omsagent/plugin/arc_k8s_cluster_identity_token.rb; source/plugins/ruby/arc_k8s_cluster_identity_token.rb; 644; root; root /opt/microsoft/omsagent/plugin/out_mdm.rb; source/plugins/ruby/out_mdm.rb; 644; root; root /opt/microsoft/omsagent/plugin/filter_cadvisor2mdm.rb; source/plugins/ruby/filter_cadvisor2mdm.rb; 644; root; root /opt/microsoft/omsagent/plugin/filter_telegraf2mdm.rb; source/plugins/ruby/filter_telegraf2mdm.rb; 644; root; root diff --git a/source/plugins/ruby/arc_k8s_cluster_identity_token.rb b/source/plugins/ruby/arc_k8s_cluster_identity_token.rb new file mode 100644 index 000000000..37b7abbde --- /dev/null +++ b/source/plugins/ruby/arc_k8s_cluster_identity_token.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +require "net/http" +require "net/https" +require "uri" +require "yajl/json_gem" +require "base64" + +class ArcK8sClusterIdentity + def initialize + @@crd_resource_uri_template = "%{kube_api_server_url}/apis/clusterconfig.azure.com/v1beta1/namespaces/azure-arc/azureclusteridentityrequests/container-insights-clusteridentityrequest" + @@secret_resource_uri_template = "%{kube_api_server_url}/api/v1/namespaces/azure-arc/secrets/%{token_secret_name}" + @token_expiry_time = Time.now + @cached_access_token = String.new + @token_secret_name = String.new + @token_secret_data_name = String.new + @token_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" + @cert_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + @http_client = get_http_client + end + + def get_cluster_identity_token + # get the cluster identity token + if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration + kube_api_server_url = get_kube_api_server_url + crd_request_uri = @@crd_resource_uri_template % { + kube_api_server_url: kube_api_server_url, + } + @service_account_token = get_service_account_token + get_request = Net::HTTP::Get.new(crd_request_uri) + get_request["Authorization"] = "Bearer #{@service_account_token}" + $log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" + get_response = @http_client.request(get_request) + $log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" + + if get_response.code.to_i == 200 + status = JSON.parse(get_response.body)["status"] + tokenReference = status["tokenReference"] + @token_expiry_time = status["expirationTime"] + @token_secret_name = status["tokenReference"]["secretName"] + @token_secret_data_name = status["tokenReference"]["dataName"] + end + + if !@token_secret_name.empty? && !@token_secret_name.empty? + # get the token from secret + secret_request_uri = @@secret_resource_uri_template % { + kube_api_server_url: kube_api_server_url, + token_secret_name: @token_secret_name, + } + + get_request = Net::HTTP::Get.new(secret_request_uri) + get_request["Authorization"] = "Bearer #{@service_account_token}" + $log.info "Making GET request to #{secret_request_uri} @ #{Time.now.utc.iso8601}" + get_response = @http_client.request(get_request) + $log.info "Got response of #{get_response.code} for #{secret_request_uri} @ #{Time.now.utc.iso8601}" + + if get_response.code.to_i == 200 + token_secret = JSON.parse(get_response.body)["data"] + cluster_identity_token = token_secret[@token_secret_data_name] + @cached_access_token = Base64.decode64(cluster_identity_token) + end + else + $log.warn ("Failed to get the cluster identity token secret") + end + end + return @cached_access_token + end + + private + + def get_service_account_token() + begin + if File.exist?(@@token_file_path) && File.readable?(@@token_file_path) + token_str = File.read(@@token_file_path).strip + return token_str + else + $log.info ("Unable to read token string from #{@@token_file_path}") + return nil + end + end + end + + def get_http_client() + kube_api_server_url = get_kube_api_server_url + host = URI.parse(kube_api_server_url) + port = URI.parse(kube_api_server_url) + http = Net::HTTP.new(host, port) + http.use_ssl = true + if !File.exist?(@cert_file_path) + raise "#{@cert_file_path} doesnt exist" + else + http.ca_file = @cert_file_path + end + http.verify_mode = OpenSSL::SSL::VERIFY_PEER + return http + end + + def get_kube_api_server_url + if ENV["KUBERNETES_SERVICE_HOST"] && ENV["KUBERNETES_PORT_443_TCP_PORT"] + return "https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}" + else + $log.warn ("Kubernetes environment variable not set KUBERNETES_SERVICE_HOST: #{ENV["KUBERNETES_SERVICE_HOST"]} KUBERNETES_PORT_443_TCP_PORT: #{ENV["KUBERNETES_PORT_443_TCP_PORT"]}. Unable to form resourceUri") + return nil + end + end +end From 87904e53fda7c4193f8f7d776646edfff15aa38b Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 28 Jul 2020 20:33:08 -0700 Subject: [PATCH 02/24] wip --- .../installer/datafiles/base_container.data | 2 +- ...y_token.rb => arc_k8s_cluster_identity.rb} | 1 - source/plugins/ruby/out_mdm.rb | 33 ++++++++++++------- 3 files changed, 22 insertions(+), 14 deletions(-) rename source/plugins/ruby/{arc_k8s_cluster_identity_token.rb => arc_k8s_cluster_identity.rb} (99%) diff --git a/build/linux/installer/datafiles/base_container.data b/build/linux/installer/datafiles/base_container.data index 960678936..be7a763bf 100644 --- a/build/linux/installer/datafiles/base_container.data +++ b/build/linux/installer/datafiles/base_container.data @@ -50,7 +50,7 @@ MAINTAINER: 'Microsoft Corporation' /opt/microsoft/omsagent/plugin/kubernetes_container_inventory.rb; source/plugins/ruby/kubernetes_container_inventory.rb; 644; root; root /opt/microsoft/omsagent/plugin/proxy_utils.rb; source/plugins/ruby/proxy_utils.rb; 644; root; root -/opt/microsoft/omsagent/plugin/arc_k8s_cluster_identity_token.rb; source/plugins/ruby/arc_k8s_cluster_identity_token.rb; 644; root; root +/opt/microsoft/omsagent/plugin/arc_k8s_cluster_identity.rb; source/plugins/ruby/arc_k8s_cluster_identity.rb; 644; root; root /opt/microsoft/omsagent/plugin/out_mdm.rb; source/plugins/ruby/out_mdm.rb; 644; root; root /opt/microsoft/omsagent/plugin/filter_cadvisor2mdm.rb; source/plugins/ruby/filter_cadvisor2mdm.rb; 644; root; root /opt/microsoft/omsagent/plugin/filter_telegraf2mdm.rb; source/plugins/ruby/filter_telegraf2mdm.rb; 644; root; root diff --git a/source/plugins/ruby/arc_k8s_cluster_identity_token.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb similarity index 99% rename from source/plugins/ruby/arc_k8s_cluster_identity_token.rb rename to source/plugins/ruby/arc_k8s_cluster_identity.rb index 37b7abbde..9c003d74e 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity_token.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -53,7 +53,6 @@ def get_cluster_identity_token $log.info "Making GET request to #{secret_request_uri} @ #{Time.now.utc.iso8601}" get_response = @http_client.request(get_request) $log.info "Got response of #{get_response.code} for #{secret_request_uri} @ #{Time.now.utc.iso8601}" - if get_response.code.to_i == 200 token_secret = JSON.parse(get_response.body)["data"] cluster_identity_token = token_secret[@token_secret_data_name] diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index d801edb9a..6c4f58308 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -16,6 +16,7 @@ def initialize require_relative "KubernetesApiClient" require_relative "ApplicationInsightsUtility" require_relative "constants" + require_relative "arc_k8s_cluster_identity" @@token_resource_url = "https://monitoring.azure.com/" @@grant_type = "client_credentials" @@ -83,21 +84,29 @@ def start @log.info "POST Request url: #{@@post_request_url}" ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMPluginStart", {}) - # Check to see if SP exists, if it does use SP. Else, use msi - sp_client_id = @data_hash["aadClientId"] - sp_client_secret = @data_hash["aadClientSecret"] - - if (!sp_client_id.nil? && !sp_client_id.empty? && sp_client_id.downcase != "msi") - @useMsi = false - aad_token_url = @@aad_token_url_template % { tenant_id: @data_hash["tenantId"] } - @parsed_token_uri = URI.parse(aad_token_url) + # arc k8s cluster uses cluster identity + if aks_resource_id.downcase.include?("microsoft.kubernetes/connectedclusters") + @log.info "using cluster identity token since cluster is azure k8s cluster" + @cluster_identity = ArcK8sClusterIdentity.new + @cached_access_token = @cluster_identity.get_cluster_identity_token else - @useMsi = true - msi_endpoint = @@msi_endpoint_template % { user_assigned_client_id: @@user_assigned_client_id, resource: @@token_resource_url } - @parsed_token_uri = URI.parse(msi_endpoint) + # Check to see if SP exists, if it does use SP. Else, use msi + sp_client_id = @data_hash["aadClientId"] + sp_client_secret = @data_hash["aadClientSecret"] + + if (!sp_client_id.nil? && !sp_client_id.empty? && sp_client_id.downcase != "msi") + @useMsi = false + aad_token_url = @@aad_token_url_template % { tenant_id: @data_hash["tenantId"] } + @parsed_token_uri = URI.parse(aad_token_url) + else + @useMsi = true + msi_endpoint = @@msi_endpoint_template % { user_assigned_client_id: @@user_assigned_client_id, resource: @@token_resource_url } + @parsed_token_uri = URI.parse(msi_endpoint) + end + + @cached_access_token = get_access_token end - @cached_access_token = get_access_token end rescue => e @log.info "exception when initializing out_mdm #{e}" From a34571f15ecbec7c797467d3f6fbc690fbea83e7 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 28 Jul 2020 21:17:46 -0700 Subject: [PATCH 03/24] fix exception --- .../ruby/arc_k8s_cluster_identity_token.rb | 105 ++++++++++++++++++ 1 file changed, 105 insertions(+) create mode 100644 source/plugins/ruby/arc_k8s_cluster_identity_token.rb diff --git a/source/plugins/ruby/arc_k8s_cluster_identity_token.rb b/source/plugins/ruby/arc_k8s_cluster_identity_token.rb new file mode 100644 index 000000000..04d94db25 --- /dev/null +++ b/source/plugins/ruby/arc_k8s_cluster_identity_token.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +require "net/http" +require "net/https" +require "uri" +require "yajl/json_gem" +require "base64" + +class ArcK8sClusterIdentity + def initialize + @@crd_resource_uri_template = "%{kube_api_server_url}/apis/clusterconfig.azure.com/v1beta1/namespaces/azure-arc/azureclusteridentityrequests/container-insights-clusteridentityrequest" + @@secret_resource_uri_template = "%{kube_api_server_url}/api/v1/namespaces/azure-arc/secrets/%{token_secret_name}" + @token_expiry_time = Time.now + @cached_access_token = String.new + @token_secret_name = String.new + @token_secret_data_name = String.new + @token_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" + @cert_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + @http_client = get_http_client + end + + def get_cluster_identity_token + # get the cluster identity token + if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration + kube_api_server_url = get_kube_api_server_url + crd_request_uri = @@crd_resource_uri_template % { + kube_api_server_url: kube_api_server_url, + } + @service_account_token = get_service_account_token + get_request = Net::HTTP::Get.new(crd_request_uri) + get_request["Authorization"] = "Bearer #{@service_account_token}" + $log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" + get_response = @http_client.request(get_request) + $log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" + + if get_response.code.to_i == 200 + status = JSON.parse(get_response.body)["status"] + tokenReference = status["tokenReference"] + @token_expiry_time = status["expirationTime"] + @token_secret_name = status["tokenReference"]["secretName"] + @token_secret_data_name = status["tokenReference"]["dataName"] + end + + if !@token_secret_name.empty? && !@token_secret_name.empty? + # get the token from secret + secret_request_uri = @@secret_resource_uri_template % { + kube_api_server_url: kube_api_server_url, + token_secret_name: @token_secret_name, + } + + get_request = Net::HTTP::Get.new(secret_request_uri) + get_request["Authorization"] = "Bearer #{@service_account_token}" + $log.info "Making GET request to #{secret_request_uri} @ #{Time.now.utc.iso8601}" + get_response = @http_client.request(get_request) + $log.info "Got response of #{get_response.code} for #{secret_request_uri} @ #{Time.now.utc.iso8601}" + if get_response.code.to_i == 200 + token_secret = JSON.parse(get_response.body)["data"] + cluster_identity_token = token_secret[@token_secret_data_name] + @cached_access_token = Base64.decode64(cluster_identity_token) + end + else + $log.warn ("Failed to get the cluster identity token secret") + end + end + return @cached_access_token + end + + private + + def get_service_account_token() + begin + if File.exist?(@token_file_path) && File.readable?(@token_file_path) + token_str = File.read(@token_file_path).strip + return token_str + else + $log.info ("Unable to read token string from #{@token_file_path}") + return nil + end + end + end + + def get_http_client() + kube_api_server_url = get_kube_api_server_url + host = URI.parse(kube_api_server_url) + port = URI.parse(kube_api_server_url) + http = Net::HTTP.new(host, port) + http.use_ssl = true + if !File.exist?(@cert_file_path) + raise "#{@cert_file_path} doesnt exist" + else + http.ca_file = @cert_file_path + end + http.verify_mode = OpenSSL::SSL::VERIFY_PEER + return http + end + + def get_kube_api_server_url + if ENV["KUBERNETES_SERVICE_HOST"] && ENV["KUBERNETES_PORT_443_TCP_PORT"] + return "https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}" + else + $log.warn ("Kubernetes environment variable not set KUBERNETES_SERVICE_HOST: #{ENV["KUBERNETES_SERVICE_HOST"]} KUBERNETES_PORT_443_TCP_PORT: #{ENV["KUBERNETES_PORT_443_TCP_PORT"]}. Unable to form resourceUri") + return nil + end + end +end From 5586f7051821466c4d8988503e046b7b217a785b Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 28 Jul 2020 21:50:27 -0700 Subject: [PATCH 04/24] fix exceptions --- .../plugins/ruby/arc_k8s_cluster_identity.rb | 24 ++-- .../ruby/arc_k8s_cluster_identity_token.rb | 105 ------------------ 2 files changed, 12 insertions(+), 117 deletions(-) delete mode 100644 source/plugins/ruby/arc_k8s_cluster_identity_token.rb diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 9c003d74e..6e40fa0fc 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -7,9 +7,11 @@ require "base64" class ArcK8sClusterIdentity + + @@crd_resource_uri_template = "%{kube_api_server_url}/apis/clusterconfig.azure.com/v1beta1/namespaces/azure-arc/azureclusteridentityrequests/container-insights-clusteridentityrequest" + @@secret_resource_uri_template = "%{kube_api_server_url}/api/v1/namespaces/azure-arc/secrets/%{token_secret_name}" + def initialize - @@crd_resource_uri_template = "%{kube_api_server_url}/apis/clusterconfig.azure.com/v1beta1/namespaces/azure-arc/azureclusteridentityrequests/container-insights-clusteridentityrequest" - @@secret_resource_uri_template = "%{kube_api_server_url}/api/v1/namespaces/azure-arc/secrets/%{token_secret_name}" @token_expiry_time = Time.now @cached_access_token = String.new @token_secret_name = String.new @@ -26,9 +28,9 @@ def get_cluster_identity_token crd_request_uri = @@crd_resource_uri_template % { kube_api_server_url: kube_api_server_url, } - @service_account_token = get_service_account_token + service_account_token = get_service_account_token get_request = Net::HTTP::Get.new(crd_request_uri) - get_request["Authorization"] = "Bearer #{@service_account_token}" + get_request["Authorization"] = "Bearer #{service_account_token}" $log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" get_response = @http_client.request(get_request) $log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" @@ -49,7 +51,7 @@ def get_cluster_identity_token } get_request = Net::HTTP::Get.new(secret_request_uri) - get_request["Authorization"] = "Bearer #{@service_account_token}" + get_request["Authorization"] = "Bearer #{service_account_token}" $log.info "Making GET request to #{secret_request_uri} @ #{Time.now.utc.iso8601}" get_response = @http_client.request(get_request) $log.info "Got response of #{get_response.code} for #{secret_request_uri} @ #{Time.now.utc.iso8601}" @@ -66,14 +68,13 @@ def get_cluster_identity_token end private - def get_service_account_token() begin - if File.exist?(@@token_file_path) && File.readable?(@@token_file_path) - token_str = File.read(@@token_file_path).strip + if File.exist?(@token_file_path) && File.readable?(@token_file_path) + token_str = File.read(@token_file_path).strip return token_str else - $log.info ("Unable to read token string from #{@@token_file_path}") + $log.info ("Unable to read token string from #{@token_file_path}") return nil end end @@ -81,9 +82,8 @@ def get_service_account_token() def get_http_client() kube_api_server_url = get_kube_api_server_url - host = URI.parse(kube_api_server_url) - port = URI.parse(kube_api_server_url) - http = Net::HTTP.new(host, port) + base_api_server_url = URI.parse(kube_api_server_url) + http = Net::HTTP.new(base_api_server_url.host, base_api_server_url.port) http.use_ssl = true if !File.exist?(@cert_file_path) raise "#{@cert_file_path} doesnt exist" diff --git a/source/plugins/ruby/arc_k8s_cluster_identity_token.rb b/source/plugins/ruby/arc_k8s_cluster_identity_token.rb deleted file mode 100644 index 04d94db25..000000000 --- a/source/plugins/ruby/arc_k8s_cluster_identity_token.rb +++ /dev/null @@ -1,105 +0,0 @@ -# frozen_string_literal: true - -require "net/http" -require "net/https" -require "uri" -require "yajl/json_gem" -require "base64" - -class ArcK8sClusterIdentity - def initialize - @@crd_resource_uri_template = "%{kube_api_server_url}/apis/clusterconfig.azure.com/v1beta1/namespaces/azure-arc/azureclusteridentityrequests/container-insights-clusteridentityrequest" - @@secret_resource_uri_template = "%{kube_api_server_url}/api/v1/namespaces/azure-arc/secrets/%{token_secret_name}" - @token_expiry_time = Time.now - @cached_access_token = String.new - @token_secret_name = String.new - @token_secret_data_name = String.new - @token_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" - @cert_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - @http_client = get_http_client - end - - def get_cluster_identity_token - # get the cluster identity token - if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration - kube_api_server_url = get_kube_api_server_url - crd_request_uri = @@crd_resource_uri_template % { - kube_api_server_url: kube_api_server_url, - } - @service_account_token = get_service_account_token - get_request = Net::HTTP::Get.new(crd_request_uri) - get_request["Authorization"] = "Bearer #{@service_account_token}" - $log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" - get_response = @http_client.request(get_request) - $log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" - - if get_response.code.to_i == 200 - status = JSON.parse(get_response.body)["status"] - tokenReference = status["tokenReference"] - @token_expiry_time = status["expirationTime"] - @token_secret_name = status["tokenReference"]["secretName"] - @token_secret_data_name = status["tokenReference"]["dataName"] - end - - if !@token_secret_name.empty? && !@token_secret_name.empty? - # get the token from secret - secret_request_uri = @@secret_resource_uri_template % { - kube_api_server_url: kube_api_server_url, - token_secret_name: @token_secret_name, - } - - get_request = Net::HTTP::Get.new(secret_request_uri) - get_request["Authorization"] = "Bearer #{@service_account_token}" - $log.info "Making GET request to #{secret_request_uri} @ #{Time.now.utc.iso8601}" - get_response = @http_client.request(get_request) - $log.info "Got response of #{get_response.code} for #{secret_request_uri} @ #{Time.now.utc.iso8601}" - if get_response.code.to_i == 200 - token_secret = JSON.parse(get_response.body)["data"] - cluster_identity_token = token_secret[@token_secret_data_name] - @cached_access_token = Base64.decode64(cluster_identity_token) - end - else - $log.warn ("Failed to get the cluster identity token secret") - end - end - return @cached_access_token - end - - private - - def get_service_account_token() - begin - if File.exist?(@token_file_path) && File.readable?(@token_file_path) - token_str = File.read(@token_file_path).strip - return token_str - else - $log.info ("Unable to read token string from #{@token_file_path}") - return nil - end - end - end - - def get_http_client() - kube_api_server_url = get_kube_api_server_url - host = URI.parse(kube_api_server_url) - port = URI.parse(kube_api_server_url) - http = Net::HTTP.new(host, port) - http.use_ssl = true - if !File.exist?(@cert_file_path) - raise "#{@cert_file_path} doesnt exist" - else - http.ca_file = @cert_file_path - end - http.verify_mode = OpenSSL::SSL::VERIFY_PEER - return http - end - - def get_kube_api_server_url - if ENV["KUBERNETES_SERVICE_HOST"] && ENV["KUBERNETES_PORT_443_TCP_PORT"] - return "https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}" - else - $log.warn ("Kubernetes environment variable not set KUBERNETES_SERVICE_HOST: #{ENV["KUBERNETES_SERVICE_HOST"]} KUBERNETES_PORT_443_TCP_PORT: #{ENV["KUBERNETES_PORT_443_TCP_PORT"]}. Unable to form resourceUri") - return nil - end - end -end From 0fcbbc43a16846fd697c79b7bcc3574d6de99df5 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 28 Jul 2020 22:19:54 -0700 Subject: [PATCH 05/24] fix exception --- source/plugins/ruby/out_mdm.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index 6c4f58308..cdd0961e1 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -46,6 +46,8 @@ def initialize @useMsi = false @metrics_flushed_count = 0 + @cluster_identity = nil + @isArcK8sCluster = false @get_access_token_backoff_expiry = Time.now end @@ -87,6 +89,7 @@ def start # arc k8s cluster uses cluster identity if aks_resource_id.downcase.include?("microsoft.kubernetes/connectedclusters") @log.info "using cluster identity token since cluster is azure k8s cluster" + @isArcK8sCluster = true @cluster_identity = ArcK8sClusterIdentity.new @cached_access_token = @cluster_identity.get_cluster_identity_token else @@ -235,7 +238,14 @@ def write(chunk) def send_to_mdm(post_body) begin - access_token = get_access_token + if (!!@isArcK8sCluster) + if !@cluster_identity.nil? + @cluster_identity = ArcK8sClusterIdentity.new + end + access_token = @cluster_identity.get_cluster_identity_token + else + access_token = get_access_token + end request = Net::HTTP::Post.new(@post_request_uri.request_uri) request["Content-Type"] = "application/x-ndjson" request["Authorization"] = "Bearer #{access_token}" From c87f1c7c99070c2fc9084d9faa1bc27d7572529f Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 28 Jul 2020 22:56:25 -0700 Subject: [PATCH 06/24] fix bug --- source/plugins/ruby/out_mdm.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index cdd0961e1..eae6779e3 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -239,7 +239,7 @@ def write(chunk) def send_to_mdm(post_body) begin if (!!@isArcK8sCluster) - if !@cluster_identity.nil? + if @cluster_identity.nil? @cluster_identity = ArcK8sClusterIdentity.new end access_token = @cluster_identity.get_cluster_identity_token From 9edac2baf125730ce5a3339c9897f94981bbe782 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 28 Jul 2020 23:25:44 -0700 Subject: [PATCH 07/24] fix bug --- source/plugins/ruby/arc_k8s_cluster_identity.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 6e40fa0fc..5ff4fd4ab 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -5,6 +5,7 @@ require "uri" require "yajl/json_gem" require "base64" +require "time" class ArcK8sClusterIdentity @@ -38,7 +39,7 @@ def get_cluster_identity_token if get_response.code.to_i == 200 status = JSON.parse(get_response.body)["status"] tokenReference = status["tokenReference"] - @token_expiry_time = status["expirationTime"] + @token_expiry_time = Time.parse(status["expirationTime"]) @token_secret_name = status["tokenReference"]["secretName"] @token_secret_data_name = status["tokenReference"]["dataName"] end From 979f0556dc904b64a00ade1d92767ed9ca841452 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 29 Jul 2020 10:56:58 -0700 Subject: [PATCH 08/24] minor update --- source/plugins/ruby/arc_k8s_cluster_identity.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 5ff4fd4ab..1246febed 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -23,7 +23,7 @@ def initialize end def get_cluster_identity_token - # get the cluster identity token + # get the cluster msi identity token which is valid for 24 hrs. if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration kube_api_server_url = get_kube_api_server_url crd_request_uri = @@crd_resource_uri_template % { From ba2da01571b53ea26a6e76574fa6a3f0e2694ace Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 29 Jul 2020 22:37:18 -0700 Subject: [PATCH 09/24] refactor the code --- .../plugins/ruby/arc_k8s_cluster_identity.rb | 177 +++++++++++++++--- 1 file changed, 148 insertions(+), 29 deletions(-) diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 1246febed..6a3d46367 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -15,59 +15,165 @@ class ArcK8sClusterIdentity def initialize @token_expiry_time = Time.now @cached_access_token = String.new - @token_secret_name = String.new - @token_secret_data_name = String.new @token_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" @cert_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" @http_client = get_http_client end - def get_cluster_identity_token - # get the cluster msi identity token which is valid for 24 hrs. - if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration + def get_token_from_secret(token_secret_name, token_secret_data_name, service_account_token) + token = nil + begin + secret_request_uri = @@secret_resource_uri_template % { + kube_api_server_url: kube_api_server_url, + token_secret_name: token_secret_name, + } + get_request = Net::HTTP::Get.new(secret_request_uri) + get_request["Authorization"] = "Bearer #{service_account_token}" + $log.info "Making GET request to #{secret_request_uri} @ #{Time.now.utc.iso8601}" + get_response = @http_client.request(get_request) + $log.info "Got response of #{get_response.code} for #{secret_request_uri} @ #{Time.now.utc.iso8601}" + if get_response.code.to_i == 200 + token_secret = JSON.parse(get_response.body)["data"] + cluster_identity_token = token_secret[token_secret_data_name] + token = Base64.decode64(cluster_identity_token) + end + rescue => err + $log.warn ("get_token_from_secret API call failed: #{err}") + end + return token + end + + def get_token_reference_from_crd(service_account_token) + tokenReference = {} + begin kube_api_server_url = get_kube_api_server_url crd_request_uri = @@crd_resource_uri_template % { kube_api_server_url: kube_api_server_url, } - service_account_token = get_service_account_token get_request = Net::HTTP::Get.new(crd_request_uri) get_request["Authorization"] = "Bearer #{service_account_token}" $log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" get_response = @http_client.request(get_request) $log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" - if get_response.code.to_i == 200 status = JSON.parse(get_response.body)["status"] - tokenReference = status["tokenReference"] - @token_expiry_time = Time.parse(status["expirationTime"]) - @token_secret_name = status["tokenReference"]["secretName"] - @token_secret_data_name = status["tokenReference"]["dataName"] + tokenReference["expirationTime"] = status["expirationTime"] + tokenReference["secretName"]= status["tokenReference"]["secretName"] + tokenReference["dataName"] = status["tokenReference"]["dataName"] + end + rescue => err + $log.warn ("get_token_reference_from_crd call failed: #{err}") + end + return tokenReference + end + + def renew_near_expiry_token(service_account_token) + begin + kube_api_server_url = get_kube_api_server_url + crd_request_uri = @@crd_resource_uri_template % { + kube_api_server_url: kube_api_server_url, + } + get_request = Net::HTTP::Get.new(crd_request_uri) + get_request["Authorization"] = "Bearer #{service_account_token}" + $log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" + get_response = @http_client.request(get_request) + $log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" + if get_response.code.to_i == 404 # NOT found + #POST + update_request = Net::HTTP::Post.new(crd_request_uri) + update_request["Content-Type"] = "application/json" + elsif get_response.code.to_i == 200 # Update == Patch + #PATCH + update_request = Net::HTTP::Patch.new(crd_request_uri) + update_request["Content-Type"] = "application/merge-patch+json" end + update_request["Authorization"] = "Bearer #{service_account_token}" + update_request_body = get_update_request_body + update_request.body = update_request_body.to_json + update_response = @http_client.request(update_request) + $log.info "Got response of #{update_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" + rescue => err + $log.warn ("renew_near_expiry_token call failed: #{err}") + end + end - if !@token_secret_name.empty? && !@token_secret_name.empty? - # get the token from secret - secret_request_uri = @@secret_resource_uri_template % { - kube_api_server_url: kube_api_server_url, - token_secret_name: @token_secret_name, - } - - get_request = Net::HTTP::Get.new(secret_request_uri) - get_request["Authorization"] = "Bearer #{service_account_token}" - $log.info "Making GET request to #{secret_request_uri} @ #{Time.now.utc.iso8601}" - get_response = @http_client.request(get_request) - $log.info "Got response of #{get_response.code} for #{secret_request_uri} @ #{Time.now.utc.iso8601}" - if get_response.code.to_i == 200 - token_secret = JSON.parse(get_response.body)["data"] - cluster_identity_token = token_secret[@token_secret_data_name] - @cached_access_token = Base64.decode64(cluster_identity_token) + def get_cluster_identity_token + begin + # get the cluster msi identity token either if its empty or near expirty. Token is valid 24 hrs. + if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration + service_account_token = get_service_account_token + # renew the token if its near expiry + if !@cached_access_token.to_s.empty? && (Time.now + 60 * 60 > @token_expiry_time) + renew_near_expiry_token(service_account_token) + # sleep 30 seconds to get the renewed token available + sleep 30 + end + tokneReference = get_token_reference_from_crd(service_account_token) + if !tokneReference.nil? && !tokneReference.empty? + @token_expiry_time = Time.parse(tokneReference["expirationTime"]) + @token_secret_name = tokneReference["secretName"] + @token_secret_data_name = tokneReference["dataName"] + # get the token from secret + token = get_token_from_secret(@token_secret_name, @token_secret_data_name, service_account_token) + if !token.nil? + @cached_access_token = token + else + $log.warn ("got token nil from secret: #{@token_secret_name}") + end end - else - $log.warn ("Failed to get the cluster identity token secret") end + rescue => err + $log.warn ("get_cluster_identity_token failed: #{err}") end return @cached_access_token end + # def get_cluster_identity_token + # # get the cluster msi identity token which is valid for 24 hrs. + # if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration + # kube_api_server_url = get_kube_api_server_url + # crd_request_uri = @@crd_resource_uri_template % { + # kube_api_server_url: kube_api_server_url, + # } + # service_account_token = get_service_account_token + # get_request = Net::HTTP::Get.new(crd_request_uri) + # get_request["Authorization"] = "Bearer #{service_account_token}" + # $log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" + # get_response = @http_client.request(get_request) + # $log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" + + # if get_response.code.to_i == 200 + # status = JSON.parse(get_response.body)["status"] + # tokenReference = status["tokenReference"] + # @token_expiry_time = Time.parse(status["expirationTime"]) + # @token_secret_name = status["tokenReference"]["secretName"] + # @token_secret_data_name = status["tokenReference"]["dataName"] + # end + + # if !@token_secret_name.empty? && !@token_secret_name.empty? + # # get the token from secret + # secret_request_uri = @@secret_resource_uri_template % { + # kube_api_server_url: kube_api_server_url, + # token_secret_name: @token_secret_name, + # } + + # get_request = Net::HTTP::Get.new(secret_request_uri) + # get_request["Authorization"] = "Bearer #{service_account_token}" + # $log.info "Making GET request to #{secret_request_uri} @ #{Time.now.utc.iso8601}" + # get_response = @http_client.request(get_request) + # $log.info "Got response of #{get_response.code} for #{secret_request_uri} @ #{Time.now.utc.iso8601}" + # if get_response.code.to_i == 200 + # token_secret = JSON.parse(get_response.body)["data"] + # cluster_identity_token = token_secret[@token_secret_data_name] + # @cached_access_token = Base64.decode64(cluster_identity_token) + # end + # else + # $log.warn ("Failed to get the cluster identity token secret") + # end + # end + # return @cached_access_token + # end + private def get_service_account_token() begin @@ -103,4 +209,17 @@ def get_kube_api_server_url return nil end end + + def get_update_request_body + body = {} + body["apiVersion"] = "clusterconfig.azure.com/v1beta1" + body["kind"] = "AzureClusterIdentityRequest" + body["metadata"] = {} + body["metadata"]["name"] = "container-insights-clusteridentityrequest" + body["metadata"]["namespace"] = "azure-arc" + body["spec"] = {} + body["spec"]["audience"] = "https://monitoring.azure.com/" + return body + end + end From fa971c5243388572617030c920878fe8aff7b26a Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 30 Jul 2020 00:41:37 -0700 Subject: [PATCH 10/24] more refactoring --- .../plugins/ruby/arc_k8s_cluster_identity.rb | 134 +++++++----------- 1 file changed, 49 insertions(+), 85 deletions(-) diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 6a3d46367..85940aea5 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -9,26 +9,35 @@ class ArcK8sClusterIdentity - @@crd_resource_uri_template = "%{kube_api_server_url}/apis/clusterconfig.azure.com/v1beta1/namespaces/azure-arc/azureclusteridentityrequests/container-insights-clusteridentityrequest" - @@secret_resource_uri_template = "%{kube_api_server_url}/api/v1/namespaces/azure-arc/secrets/%{token_secret_name}" + @@cluster_config_crd_api_version = "clusterconfig.azure.com/v1beta1" + @@cluster_identity_resource_name = "container-insights-clusteridentityrequest" + @@cluster_identity_resource_namespace = "azure-arc" + @@cluster_identity_token_secret_namespace = "azure-arc" + @@crd_resource_uri_template = "%{kube_api_server_url}/apis/%{cluster_config_crd_api_version}/namespaces/%{cluster_identity_resource_namespace}/azureclusteridentityrequests/%{cluster_identity_resource_name}" + @@secret_resource_uri_template = "%{kube_api_server_url}/api/v1/namespaces/%{cluster_identity_token_secret_namespace}/secrets/%{token_secret_name}" + @@azure_monitor_custom_metrics_audience = "https://monitoring.azure.com/" + @@cluster_identity_request_kind = "AzureClusterIdentityRequest" def initialize @token_expiry_time = Time.now @cached_access_token = String.new @token_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" @cert_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + @kube_api_server_url = get_kube_api_server_url @http_client = get_http_client + @service_account_token = get_service_account_token end - def get_token_from_secret(token_secret_name, token_secret_data_name, service_account_token) + def get_token_from_secret(token_secret_name, token_secret_data_name) token = nil begin secret_request_uri = @@secret_resource_uri_template % { - kube_api_server_url: kube_api_server_url, + kube_api_server_url: @kube_api_server_url, + cluster_identity_token_secret_namespace: @@cluster_identity_token_secret_namespace, token_secret_name: token_secret_name, } get_request = Net::HTTP::Get.new(secret_request_uri) - get_request["Authorization"] = "Bearer #{service_account_token}" + get_request["Authorization"] = "Bearer #{@service_account_token}" $log.info "Making GET request to #{secret_request_uri} @ #{Time.now.utc.iso8601}" get_response = @http_client.request(get_request) $log.info "Got response of #{get_response.code} for #{secret_request_uri} @ #{Time.now.utc.iso8601}" @@ -43,15 +52,16 @@ def get_token_from_secret(token_secret_name, token_secret_data_name, service_acc return token end - def get_token_reference_from_crd(service_account_token) + def get_token_reference_from_crd() tokenReference = {} begin - kube_api_server_url = get_kube_api_server_url crd_request_uri = @@crd_resource_uri_template % { - kube_api_server_url: kube_api_server_url, + kube_api_server_url: @kube_api_server_url, + cluster_config_crd_api_version: @@cluster_config_crd_api_version, + cluster_identity_resource_namespace: @@cluster_identity_resource_namespace, } get_request = Net::HTTP::Get.new(crd_request_uri) - get_request["Authorization"] = "Bearer #{service_account_token}" + get_request["Authorization"] = "Bearer #{@service_account_token}" $log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" get_response = @http_client.request(get_request) $log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" @@ -67,14 +77,15 @@ def get_token_reference_from_crd(service_account_token) return tokenReference end - def renew_near_expiry_token(service_account_token) + def renew_near_expiry_token() begin - kube_api_server_url = get_kube_api_server_url crd_request_uri = @@crd_resource_uri_template % { - kube_api_server_url: kube_api_server_url, + kube_api_server_url: @kube_api_server_url, + cluster_config_crd_api_version: @@cluster_config_crd_api_version, + cluster_identity_resource_namespace: @@cluster_identity_resource_namespace, } get_request = Net::HTTP::Get.new(crd_request_uri) - get_request["Authorization"] = "Bearer #{service_account_token}" + get_request["Authorization"] = "Bearer #{@service_account_token}" $log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" get_response = @http_client.request(get_request) $log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" @@ -87,7 +98,7 @@ def renew_near_expiry_token(service_account_token) update_request = Net::HTTP::Patch.new(crd_request_uri) update_request["Content-Type"] = "application/merge-patch+json" end - update_request["Authorization"] = "Bearer #{service_account_token}" + update_request["Authorization"] = "Bearer #{@service_account_token}" update_request_body = get_update_request_body update_request.body = update_request_body.to_json update_response = @http_client.request(update_request) @@ -97,30 +108,29 @@ def renew_near_expiry_token(service_account_token) end end - def get_cluster_identity_token + def get_cluster_identity_token() begin # get the cluster msi identity token either if its empty or near expirty. Token is valid 24 hrs. if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration - service_account_token = get_service_account_token - # renew the token if its near expiry - if !@cached_access_token.to_s.empty? && (Time.now + 60 * 60 > @token_expiry_time) - renew_near_expiry_token(service_account_token) - # sleep 30 seconds to get the renewed token available - sleep 30 - end - tokneReference = get_token_reference_from_crd(service_account_token) - if !tokneReference.nil? && !tokneReference.empty? - @token_expiry_time = Time.parse(tokneReference["expirationTime"]) - @token_secret_name = tokneReference["secretName"] - @token_secret_data_name = tokneReference["dataName"] - # get the token from secret - token = get_token_from_secret(@token_secret_name, @token_secret_data_name, service_account_token) - if !token.nil? - @cached_access_token = token - else - $log.warn ("got token nil from secret: #{@token_secret_name}") + # renew the token if its near expiry + if !@cached_access_token.to_s.empty? && (Time.now + 60 * 60 > @token_expiry_time) + renew_near_expiry_token + # sleep 30 seconds to get the renewed token available + sleep 30 + end + tokneReference = get_token_reference_from_crd + if !tokneReference.nil? && !tokneReference.empty? + @token_expiry_time = Time.parse(tokneReference["expirationTime"]) + token_secret_name = tokneReference["secretName"] + token_secret_data_name = tokneReference["dataName"] + # get the token from secret + token = get_token_from_secret(token_secret_name, token_secret_data_name) + if !token.nil? + @cached_access_token = token + else + $log.warn ("got token nil from secret: #{@token_secret_name}") + end end - end end rescue => err $log.warn ("get_cluster_identity_token failed: #{err}") @@ -128,52 +138,6 @@ def get_cluster_identity_token return @cached_access_token end - # def get_cluster_identity_token - # # get the cluster msi identity token which is valid for 24 hrs. - # if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration - # kube_api_server_url = get_kube_api_server_url - # crd_request_uri = @@crd_resource_uri_template % { - # kube_api_server_url: kube_api_server_url, - # } - # service_account_token = get_service_account_token - # get_request = Net::HTTP::Get.new(crd_request_uri) - # get_request["Authorization"] = "Bearer #{service_account_token}" - # $log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" - # get_response = @http_client.request(get_request) - # $log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" - - # if get_response.code.to_i == 200 - # status = JSON.parse(get_response.body)["status"] - # tokenReference = status["tokenReference"] - # @token_expiry_time = Time.parse(status["expirationTime"]) - # @token_secret_name = status["tokenReference"]["secretName"] - # @token_secret_data_name = status["tokenReference"]["dataName"] - # end - - # if !@token_secret_name.empty? && !@token_secret_name.empty? - # # get the token from secret - # secret_request_uri = @@secret_resource_uri_template % { - # kube_api_server_url: kube_api_server_url, - # token_secret_name: @token_secret_name, - # } - - # get_request = Net::HTTP::Get.new(secret_request_uri) - # get_request["Authorization"] = "Bearer #{service_account_token}" - # $log.info "Making GET request to #{secret_request_uri} @ #{Time.now.utc.iso8601}" - # get_response = @http_client.request(get_request) - # $log.info "Got response of #{get_response.code} for #{secret_request_uri} @ #{Time.now.utc.iso8601}" - # if get_response.code.to_i == 200 - # token_secret = JSON.parse(get_response.body)["data"] - # cluster_identity_token = token_secret[@token_secret_data_name] - # @cached_access_token = Base64.decode64(cluster_identity_token) - # end - # else - # $log.warn ("Failed to get the cluster identity token secret") - # end - # end - # return @cached_access_token - # end - private def get_service_account_token() begin @@ -212,13 +176,13 @@ def get_kube_api_server_url def get_update_request_body body = {} - body["apiVersion"] = "clusterconfig.azure.com/v1beta1" - body["kind"] = "AzureClusterIdentityRequest" + body["apiVersion"] = @@cluster_config_crd_api_version + body["kind"] = @@cluster_identity_request_kind body["metadata"] = {} - body["metadata"]["name"] = "container-insights-clusteridentityrequest" - body["metadata"]["namespace"] = "azure-arc" + body["metadata"]["name"] = @@cluster_identity_resource_name + body["metadata"]["namespace"] = @@cluster_identity_resource_namespace body["spec"] = {} - body["spec"]["audience"] = "https://monitoring.azure.com/" + body["spec"]["audience"] = @@azure_monitor_custom_metrics_audience return body end From 4a4573720a70a4c99c3cee56b7b430b70babe228 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 30 Jul 2020 01:15:32 -0700 Subject: [PATCH 11/24] fix bug --- source/plugins/ruby/arc_k8s_cluster_identity.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 85940aea5..172a5d74d 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -59,6 +59,7 @@ def get_token_reference_from_crd() kube_api_server_url: @kube_api_server_url, cluster_config_crd_api_version: @@cluster_config_crd_api_version, cluster_identity_resource_namespace: @@cluster_identity_resource_namespace, + cluster_identity_resource_name: @@cluster_identity_resource_name, } get_request = Net::HTTP::Get.new(crd_request_uri) get_request["Authorization"] = "Bearer #{@service_account_token}" @@ -83,6 +84,7 @@ def renew_near_expiry_token() kube_api_server_url: @kube_api_server_url, cluster_config_crd_api_version: @@cluster_config_crd_api_version, cluster_identity_resource_namespace: @@cluster_identity_resource_namespace, + cluster_identity_resource_name: @@cluster_identity_resource_name, } get_request = Net::HTTP::Get.new(crd_request_uri) get_request["Authorization"] = "Bearer #{@service_account_token}" From 414f487c572f7a98b5162f8da9a7fb2a2e09c650 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 30 Jul 2020 01:32:12 -0700 Subject: [PATCH 12/24] typo fix --- source/plugins/ruby/out_mdm.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index eae6779e3..1f428f2b2 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -88,7 +88,7 @@ def start # arc k8s cluster uses cluster identity if aks_resource_id.downcase.include?("microsoft.kubernetes/connectedclusters") - @log.info "using cluster identity token since cluster is azure k8s cluster" + @log.info "using cluster identity token since cluster is azure arc k8s cluster" @isArcK8sCluster = true @cluster_identity = ArcK8sClusterIdentity.new @cached_access_token = @cluster_identity.get_cluster_identity_token From 9fddedfff11edb6b2e15553bd3c4cf996417aa53 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 30 Jul 2020 10:07:22 -0700 Subject: [PATCH 13/24] fix typo --- source/plugins/ruby/arc_k8s_cluster_identity.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 172a5d74d..3238dbe31 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -120,11 +120,11 @@ def get_cluster_identity_token() # sleep 30 seconds to get the renewed token available sleep 30 end - tokneReference = get_token_reference_from_crd - if !tokneReference.nil? && !tokneReference.empty? - @token_expiry_time = Time.parse(tokneReference["expirationTime"]) - token_secret_name = tokneReference["secretName"] - token_secret_data_name = tokneReference["dataName"] + tokenReference = get_token_reference_from_crd + if !tokenReference.nil? && !tokenReference.empty? + @token_expiry_time = Time.parse(tokenReference["expirationTime"]) + token_secret_name = tokenReference["secretName"] + token_secret_data_name = tokenReference["dataName"] # get the token from secret token = get_token_from_secret(token_secret_name, token_secret_data_name) if !token.nil? From f8419bc7d40e090d628e97c4853ad54980c16855 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 31 Jul 2020 00:34:13 -0700 Subject: [PATCH 14/24] wait for 1min after token renewal request --- source/plugins/ruby/arc_k8s_cluster_identity.rb | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 3238dbe31..490b68685 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -116,9 +116,10 @@ def get_cluster_identity_token() if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration # renew the token if its near expiry if !@cached_access_token.to_s.empty? && (Time.now + 60 * 60 > @token_expiry_time) + $log.info ("renewing the token since its near expiry") renew_near_expiry_token - # sleep 30 seconds to get the renewed token available - sleep 30 + # sleep 60 seconds to get the renewed token available + sleep 60 end tokenReference = get_token_reference_from_crd if !tokenReference.nil? && !tokenReference.empty? @@ -132,6 +133,8 @@ def get_cluster_identity_token() else $log.warn ("got token nil from secret: #{@token_secret_name}") end + else + $log.warn ("got token reference either nil or empty") end end rescue => err @@ -147,7 +150,7 @@ def get_service_account_token() token_str = File.read(@token_file_path).strip return token_str else - $log.info ("Unable to read token string from #{@token_file_path}") + $log.warn ("Unable to read token string from #{@token_file_path}") return nil end end From 8dbc28acf2cf13edcedf3658fe2e769fbc23f6f2 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 1 Aug 2020 20:02:56 -0700 Subject: [PATCH 15/24] add proxy support for arc k8s mdm endpoint --- source/plugins/ruby/out_mdm.rb | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index 1f428f2b2..cf9c6f064 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -17,6 +17,7 @@ def initialize require_relative "ApplicationInsightsUtility" require_relative "constants" require_relative "arc_k8s_cluster_identity" + require_relative "proxy_utils" @@token_resource_url = "https://monitoring.azure.com/" @@grant_type = "client_credentials" @@ -79,17 +80,29 @@ def start if @can_send_data_to_mdm @log.info "MDM Metrics supported in #{aks_region} region" + if aks_resource_id.downcase.include?("microsoft.kubernetes/connectedclusters") + @isArcK8sCluster = true + end @@post_request_url = @@post_request_url_template % { aks_region: aks_region, aks_resource_id: aks_resource_id } @post_request_uri = URI.parse(@@post_request_url) - @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port) + if (!!@isArcK8sCluster) + proxy = (ProxyUtils.getProxyConfiguration) + if proxy.nil? || proxy.empty? + @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port) + else + @log.info "Proxy configured on this cluster: #{aks_resource_id}" + @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port, proxy[:addr], proxy[:port], proxy[:user], proxy[:pass]) + end + else + @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port) + end @http_client.use_ssl = true @log.info "POST Request url: #{@@post_request_url}" ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMPluginStart", {}) # arc k8s cluster uses cluster identity - if aks_resource_id.downcase.include?("microsoft.kubernetes/connectedclusters") + if (!!@isArcK8sCluster) @log.info "using cluster identity token since cluster is azure arc k8s cluster" - @isArcK8sCluster = true @cluster_identity = ArcK8sClusterIdentity.new @cached_access_token = @cluster_identity.get_cluster_identity_token else From d0c4c97174f2387a56e3bee2dc25f2d7bf124f84 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 10 Aug 2020 13:18:49 -0700 Subject: [PATCH 16/24] avoid additional get call --- build/linux/Makefile | 2 +- .../templates/omsagent-arc-k8s-crd.yaml | 9 ++ .../templates/omsagent-rbac.yaml | 8 ++ .../build-and-publish-docker-image.sh | 2 +- .../plugins/ruby/arc_k8s_cluster_identity.rb | 85 ++++++++++--------- source/plugins/ruby/out_mdm.rb | 27 +++--- 6 files changed, 78 insertions(+), 55 deletions(-) create mode 100644 charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml diff --git a/build/linux/Makefile b/build/linux/Makefile index 0a20ed205..3f35e1204 100644 --- a/build/linux/Makefile +++ b/build/linux/Makefile @@ -118,7 +118,7 @@ distclean : clean PROVIDER_STATUS: @echo "========================= Performing Building provider" @echo "clean up everything under: $(INTERMEDIATE_BASE_DIR) to avoid picking up old binaries" - $(RMDIR) $(INTERMEDIATE_BASE_DIR) + sudo $(RMDIR) $(INTERMEDIATE_BASE_DIR) KIT_STATUS: @echo "========================= Performing Building provider tests" diff --git a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml new file mode 100644 index 000000000..596a29e7a --- /dev/null +++ b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml @@ -0,0 +1,9 @@ +{{- if contains "microsoft.kubernetes/connectedclusters" (.Values.omsagent.env.clusterId | lower) }} +apiVersion: clusterconfig.azure.com/v1beta1 +kind: AzureClusterIdentityRequest +metadata: + name: container-insights-clusteridentityrequest + namespace: azure-arc +spec: + audience: https://monitoring.azure.com/ +{{- end }} \ No newline at end of file diff --git a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml index 9903f41ff..876d64a9f 100644 --- a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml @@ -27,8 +27,16 @@ rules: - apiGroups: ["azmon.container.insights"] resources: ["healthstates"] verbs: ["get", "create", "patch"] +- apiGroups: ["clusterconfig.azure.com"] + resources: ["azureclusteridentityrequests"] + resourceNames: ["container-insights-clusteridentityrequest"] + verbs: ["get", "create", "patch", "update"] - nonResourceURLs: ["/metrics"] verbs: ["get"] +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["container-insights-clusteridentityrequest-token"] + verbs: ["list", "get", "watch"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/kubernetes/linux/dockerbuild/build-and-publish-docker-image.sh b/kubernetes/linux/dockerbuild/build-and-publish-docker-image.sh index 982c8c491..267f15f32 100644 --- a/kubernetes/linux/dockerbuild/build-and-publish-docker-image.sh +++ b/kubernetes/linux/dockerbuild/build-and-publish-docker-image.sh @@ -127,7 +127,7 @@ baseDir=$(dirname $kubernetsDir) buildDir=$baseDir/build/linux dockerFileDir=$baseDir/kubernetes/linux -echo "sour code base directory: $baseDir" +echo "source code base directory: $baseDir" echo "build directory for docker provider: $buildDir" echo "docker file directory: $dockerFileDir" diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 490b68685..96e07b390 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -8,7 +8,7 @@ require "time" class ArcK8sClusterIdentity - + # this arc k8s crd version and arc k8s uses corresponding version v1beta1 vs v1 based on the k8s version for apiextensions.k8s.io @@cluster_config_crd_api_version = "clusterconfig.azure.com/v1beta1" @@cluster_identity_resource_name = "container-insights-clusteridentityrequest" @@cluster_identity_resource_namespace = "azure-arc" @@ -28,6 +28,8 @@ def initialize @service_account_token = get_service_account_token end + private + def get_token_from_secret(token_secret_name, token_secret_data_name) token = nil begin @@ -52,6 +54,8 @@ def get_token_from_secret(token_secret_name, token_secret_data_name) return token end + private + def get_token_reference_from_crd() tokenReference = {} begin @@ -69,7 +73,7 @@ def get_token_reference_from_crd() if get_response.code.to_i == 200 status = JSON.parse(get_response.body)["status"] tokenReference["expirationTime"] = status["expirationTime"] - tokenReference["secretName"]= status["tokenReference"]["secretName"] + tokenReference["secretName"] = status["tokenReference"]["secretName"] tokenReference["dataName"] = status["tokenReference"]["dataName"] end rescue => err @@ -78,6 +82,8 @@ def get_token_reference_from_crd() return tokenReference end + private + def renew_near_expiry_token() begin crd_request_uri = @@crd_resource_uri_template % { @@ -86,25 +92,20 @@ def renew_near_expiry_token() cluster_identity_resource_namespace: @@cluster_identity_resource_namespace, cluster_identity_resource_name: @@cluster_identity_resource_name, } - get_request = Net::HTTP::Get.new(crd_request_uri) - get_request["Authorization"] = "Bearer #{@service_account_token}" - $log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" - get_response = @http_client.request(get_request) - $log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" - if get_response.code.to_i == 404 # NOT found - #POST - update_request = Net::HTTP::Post.new(crd_request_uri) - update_request["Content-Type"] = "application/json" - elsif get_response.code.to_i == 200 # Update == Patch - #PATCH - update_request = Net::HTTP::Patch.new(crd_request_uri) - update_request["Content-Type"] = "application/merge-patch+json" - end + update_request = Net::HTTP::Patch.new(crd_request_uri) + update_request["Content-Type"] = "application/merge-patch+json" update_request["Authorization"] = "Bearer #{@service_account_token}" update_request_body = get_update_request_body update_request.body = update_request_body.to_json update_response = @http_client.request(update_request) - $log.info "Got response of #{update_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" + $log.info "Got response of #{update_response.code} for PATCH #{crd_request_uri} @ #{Time.now.utc.iso8601}" + if update_response.code.to_i == 404 + $log.info "since crd resource doesnt exist since creating crd resource : #{@@cluster_identity_resource_name} @ #{Time.now.utc.iso8601}" + update_request = Net::HTTP::Post.new(crd_request_uri) + update_request["Content-Type"] = "application/json" + update_response = @http_client.request(update_request) + $log.info "Got response of #{update_response.code} for POST #{crd_request_uri} @ #{Time.now.utc.iso8601}" + end rescue => err $log.warn ("renew_near_expiry_token call failed: #{err}") end @@ -114,28 +115,28 @@ def get_cluster_identity_token() begin # get the cluster msi identity token either if its empty or near expirty. Token is valid 24 hrs. if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration - # renew the token if its near expiry - if !@cached_access_token.to_s.empty? && (Time.now + 60 * 60 > @token_expiry_time) - $log.info ("renewing the token since its near expiry") - renew_near_expiry_token - # sleep 60 seconds to get the renewed token available - sleep 60 - end - tokenReference = get_token_reference_from_crd - if !tokenReference.nil? && !tokenReference.empty? - @token_expiry_time = Time.parse(tokenReference["expirationTime"]) - token_secret_name = tokenReference["secretName"] - token_secret_data_name = tokenReference["dataName"] - # get the token from secret - token = get_token_from_secret(token_secret_name, token_secret_data_name) - if !token.nil? - @cached_access_token = token - else - $log.warn ("got token nil from secret: #{@token_secret_name}") - end + # renew the token if its near expiry + if !@cached_access_token.to_s.empty? && (Time.now + 60 * 60 > @token_expiry_time) + $log.info ("renewing the token since its near expiry") + renew_near_expiry_token + # sleep 60 seconds to get the renewed token available + sleep 60 + end + tokenReference = get_token_reference_from_crd + if !tokenReference.nil? && !tokenReference.empty? + @token_expiry_time = Time.parse(tokenReference["expirationTime"]) + token_secret_name = tokenReference["secretName"] + token_secret_data_name = tokenReference["dataName"] + # get the token from secret + token = get_token_from_secret(token_secret_name, token_secret_data_name) + if !token.nil? + @cached_access_token = token else - $log.warn ("got token reference either nil or empty") + $log.warn ("got token nil from secret: #{@token_secret_name}") end + else + $log.warn ("got token reference either nil or empty") + end end rescue => err $log.warn ("get_cluster_identity_token failed: #{err}") @@ -144,6 +145,7 @@ def get_cluster_identity_token() end private + def get_service_account_token() begin if File.exist?(@token_file_path) && File.readable?(@token_file_path) @@ -156,6 +158,8 @@ def get_service_account_token() end end + private + def get_http_client() kube_api_server_url = get_kube_api_server_url base_api_server_url = URI.parse(kube_api_server_url) @@ -170,6 +174,8 @@ def get_http_client() return http end + private + def get_kube_api_server_url if ENV["KUBERNETES_SERVICE_HOST"] && ENV["KUBERNETES_PORT_443_TCP_PORT"] return "https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}" @@ -179,16 +185,17 @@ def get_kube_api_server_url end end + private + def get_update_request_body body = {} body["apiVersion"] = @@cluster_config_crd_api_version body["kind"] = @@cluster_identity_request_kind body["metadata"] = {} body["metadata"]["name"] = @@cluster_identity_resource_name - body["metadata"]["namespace"] = @@cluster_identity_resource_namespace + body["metadata"]["namespace"] = @@cluster_identity_resource_namespace body["spec"] = {} body["spec"]["audience"] = @@azure_monitor_custom_metrics_audience return body end - end diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index cf9c6f064..b28c17034 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -86,13 +86,13 @@ def start @@post_request_url = @@post_request_url_template % { aks_region: aks_region, aks_resource_id: aks_resource_id } @post_request_uri = URI.parse(@@post_request_url) if (!!@isArcK8sCluster) - proxy = (ProxyUtils.getProxyConfiguration) - if proxy.nil? || proxy.empty? - @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port) - else - @log.info "Proxy configured on this cluster: #{aks_resource_id}" - @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port, proxy[:addr], proxy[:port], proxy[:user], proxy[:pass]) - end + proxy = (ProxyUtils.getProxyConfiguration) + if proxy.nil? || proxy.empty? + @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port) + else + @log.info "Proxy configured on this cluster: #{aks_resource_id}" + @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port, proxy[:addr], proxy[:port], proxy[:user], proxy[:pass]) + end else @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port) end @@ -102,9 +102,9 @@ def start # arc k8s cluster uses cluster identity if (!!@isArcK8sCluster) - @log.info "using cluster identity token since cluster is azure arc k8s cluster" - @cluster_identity = ArcK8sClusterIdentity.new - @cached_access_token = @cluster_identity.get_cluster_identity_token + @log.info "using cluster identity token since cluster is azure arc k8s cluster" + @cluster_identity = ArcK8sClusterIdentity.new + @cached_access_token = @cluster_identity.get_cluster_identity_token else # Check to see if SP exists, if it does use SP. Else, use msi sp_client_id = @data_hash["aadClientId"] @@ -122,7 +122,6 @@ def start @cached_access_token = get_access_token end - end rescue => e @log.info "exception when initializing out_mdm #{e}" @@ -252,9 +251,9 @@ def write(chunk) def send_to_mdm(post_body) begin if (!!@isArcK8sCluster) - if @cluster_identity.nil? - @cluster_identity = ArcK8sClusterIdentity.new - end + if @cluster_identity.nil? + @cluster_identity = ArcK8sClusterIdentity.new + end access_token = @cluster_identity.get_cluster_identity_token else access_token = get_access_token From ebf9452a1469b08f3073fa172f742e263898afce Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 10 Aug 2020 13:40:38 -0700 Subject: [PATCH 17/24] minor line ending fix --- .../azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml index 596a29e7a..f7873de40 100644 --- a/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-arc-k8s-crd.yaml @@ -6,4 +6,4 @@ metadata: namespace: azure-arc spec: audience: https://monitoring.azure.com/ -{{- end }} \ No newline at end of file +{{- end }} From 612a5a5846fe7dd0e4cdcc6cb4718c1f1bc03282 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 10 Aug 2020 15:41:24 -0700 Subject: [PATCH 18/24] wip --- .../plugins/ruby/arc_k8s_cluster_identity.rb | 66 +++++++++---------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 96e07b390..ddc39ef9d 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -28,6 +28,39 @@ def initialize @service_account_token = get_service_account_token end + def get_cluster_identity_token() + begin + # get the cluster msi identity token either if its empty or near expirty. Token is valid 24 hrs. + if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration + # renew the token if its near expiry + if !@cached_access_token.to_s.empty? && (Time.now + 60 * 60 > @token_expiry_time) + $log.info ("renewing the token since its near expiry") + renew_near_expiry_token + # sleep 60 seconds to get the renewed token available + sleep 60 + end + tokenReference = get_token_reference_from_crd + if !tokenReference.nil? && !tokenReference.empty? + @token_expiry_time = Time.parse(tokenReference["expirationTime"]) + token_secret_name = tokenReference["secretName"] + token_secret_data_name = tokenReference["dataName"] + # get the token from secret + token = get_token_from_secret(token_secret_name, token_secret_data_name) + if !token.nil? + @cached_access_token = token + else + $log.warn ("got token nil from secret: #{@token_secret_name}") + end + else + $log.warn ("got token reference either nil or empty") + end + end + rescue => err + $log.warn ("get_cluster_identity_token failed: #{err}") + end + return @cached_access_token + end + private def get_token_from_secret(token_secret_name, token_secret_data_name) @@ -111,39 +144,6 @@ def renew_near_expiry_token() end end - def get_cluster_identity_token() - begin - # get the cluster msi identity token either if its empty or near expirty. Token is valid 24 hrs. - if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration - # renew the token if its near expiry - if !@cached_access_token.to_s.empty? && (Time.now + 60 * 60 > @token_expiry_time) - $log.info ("renewing the token since its near expiry") - renew_near_expiry_token - # sleep 60 seconds to get the renewed token available - sleep 60 - end - tokenReference = get_token_reference_from_crd - if !tokenReference.nil? && !tokenReference.empty? - @token_expiry_time = Time.parse(tokenReference["expirationTime"]) - token_secret_name = tokenReference["secretName"] - token_secret_data_name = tokenReference["dataName"] - # get the token from secret - token = get_token_from_secret(token_secret_name, token_secret_data_name) - if !token.nil? - @cached_access_token = token - else - $log.warn ("got token nil from secret: #{@token_secret_name}") - end - else - $log.warn ("got token reference either nil or empty") - end - end - rescue => err - $log.warn ("get_cluster_identity_token failed: #{err}") - end - return @cached_access_token - end - private def get_service_account_token() From 5be8f2d204b7373917a8493a277e7266692ca20b Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 10 Aug 2020 23:52:09 -0700 Subject: [PATCH 19/24] have separate log for arc k8s cluster identity --- .../installer/datafiles/base_container.data | 4 ++ .../plugins/ruby/arc_k8s_cluster_identity.rb | 40 +++++++++++-------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/build/linux/installer/datafiles/base_container.data b/build/linux/installer/datafiles/base_container.data index be7a763bf..37ff314c3 100644 --- a/build/linux/installer/datafiles/base_container.data +++ b/build/linux/installer/datafiles/base_container.data @@ -275,6 +275,10 @@ touch /var/opt/microsoft/docker-cimprov/log/fluent_forward_failed.log chmod 666 /var/opt/microsoft/docker-cimprov/log/fluent_forward_failed.log chown omsagent:omiusers /var/opt/microsoft/docker-cimprov/log/fluent_forward_failed.log +touch /var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log +chmod 666 /var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log +chown omsagent:omiusers /var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log + mv /etc/opt/microsoft/docker-cimprov/container.conf /etc/opt/microsoft/omsagent/sysconf/omsagent.d/container.conf chown omsagent:omsagent /etc/opt/microsoft/omsagent/sysconf/omsagent.d/container.conf diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index ddc39ef9d..95136381b 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true - +require "logger" require "net/http" require "net/https" require "uri" @@ -19,6 +19,9 @@ class ArcK8sClusterIdentity @@cluster_identity_request_kind = "AzureClusterIdentityRequest" def initialize + @LogPath = "/var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log" + @log = Logger.new(@LogPath, 1, 5000000) + @log.info "initialize start @ #{Time.now.utc.iso8601}" @token_expiry_time = Time.now @cached_access_token = String.new @token_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" @@ -26,6 +29,7 @@ def initialize @kube_api_server_url = get_kube_api_server_url @http_client = get_http_client @service_account_token = get_service_account_token + @log.info "initialize complete @ #{Time.now.utc.iso8601}" end def get_cluster_identity_token() @@ -34,29 +38,31 @@ def get_cluster_identity_token() if @cached_access_token.to_s.empty? || (Time.now + 60 * 60 > @token_expiry_time) # Refresh token 1 hr from expiration # renew the token if its near expiry if !@cached_access_token.to_s.empty? && (Time.now + 60 * 60 > @token_expiry_time) - $log.info ("renewing the token since its near expiry") + @log.info "renewing the token since its near expiry @ #{Time.now.utc.iso8601}" renew_near_expiry_token # sleep 60 seconds to get the renewed token available sleep 60 end + @log.info "get token reference from crd @ #{Time.now.utc.iso8601}" tokenReference = get_token_reference_from_crd if !tokenReference.nil? && !tokenReference.empty? @token_expiry_time = Time.parse(tokenReference["expirationTime"]) token_secret_name = tokenReference["secretName"] token_secret_data_name = tokenReference["dataName"] # get the token from secret + @log.info "get token from secret @ #{Time.now.utc.iso8601}" token = get_token_from_secret(token_secret_name, token_secret_data_name) if !token.nil? @cached_access_token = token else - $log.warn ("got token nil from secret: #{@token_secret_name}") + @log.warn "got token nil from secret: #{@token_secret_name}" end else - $log.warn ("got token reference either nil or empty") + @log.warn "got token reference either nil or empty" end end rescue => err - $log.warn ("get_cluster_identity_token failed: #{err}") + @log.warn "get_cluster_identity_token failed: #{err}" end return @cached_access_token end @@ -73,16 +79,16 @@ def get_token_from_secret(token_secret_name, token_secret_data_name) } get_request = Net::HTTP::Get.new(secret_request_uri) get_request["Authorization"] = "Bearer #{@service_account_token}" - $log.info "Making GET request to #{secret_request_uri} @ #{Time.now.utc.iso8601}" + @log.info "Making GET request to #{secret_request_uri} @ #{Time.now.utc.iso8601}" get_response = @http_client.request(get_request) - $log.info "Got response of #{get_response.code} for #{secret_request_uri} @ #{Time.now.utc.iso8601}" + @log.info "Got response of #{get_response.code} for #{secret_request_uri} @ #{Time.now.utc.iso8601}" if get_response.code.to_i == 200 token_secret = JSON.parse(get_response.body)["data"] cluster_identity_token = token_secret[token_secret_data_name] token = Base64.decode64(cluster_identity_token) end rescue => err - $log.warn ("get_token_from_secret API call failed: #{err}") + @log.warn "get_token_from_secret API call failed: #{err}" end return token end @@ -100,9 +106,9 @@ def get_token_reference_from_crd() } get_request = Net::HTTP::Get.new(crd_request_uri) get_request["Authorization"] = "Bearer #{@service_account_token}" - $log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" + @log.info "Making GET request to #{crd_request_uri} @ #{Time.now.utc.iso8601}" get_response = @http_client.request(get_request) - $log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" + @log.info "Got response of #{get_response.code} for #{crd_request_uri} @ #{Time.now.utc.iso8601}" if get_response.code.to_i == 200 status = JSON.parse(get_response.body)["status"] tokenReference["expirationTime"] = status["expirationTime"] @@ -110,7 +116,7 @@ def get_token_reference_from_crd() tokenReference["dataName"] = status["tokenReference"]["dataName"] end rescue => err - $log.warn ("get_token_reference_from_crd call failed: #{err}") + @log.warn "get_token_reference_from_crd call failed: #{err}" end return tokenReference end @@ -131,16 +137,16 @@ def renew_near_expiry_token() update_request_body = get_update_request_body update_request.body = update_request_body.to_json update_response = @http_client.request(update_request) - $log.info "Got response of #{update_response.code} for PATCH #{crd_request_uri} @ #{Time.now.utc.iso8601}" + @log.info "Got response of #{update_response.code} for PATCH #{crd_request_uri} @ #{Time.now.utc.iso8601}" if update_response.code.to_i == 404 - $log.info "since crd resource doesnt exist since creating crd resource : #{@@cluster_identity_resource_name} @ #{Time.now.utc.iso8601}" + @log.info "since crd resource doesnt exist since creating crd resource : #{@@cluster_identity_resource_name} @ #{Time.now.utc.iso8601}" update_request = Net::HTTP::Post.new(crd_request_uri) update_request["Content-Type"] = "application/json" update_response = @http_client.request(update_request) - $log.info "Got response of #{update_response.code} for POST #{crd_request_uri} @ #{Time.now.utc.iso8601}" + @log.info "Got response of #{update_response.code} for POST #{crd_request_uri} @ #{Time.now.utc.iso8601}" end rescue => err - $log.warn ("renew_near_expiry_token call failed: #{err}") + @log.warn "renew_near_expiry_token call failed: #{err}" end end @@ -152,7 +158,7 @@ def get_service_account_token() token_str = File.read(@token_file_path).strip return token_str else - $log.warn ("Unable to read token string from #{@token_file_path}") + @log.warn "Unable to read token string from #{@token_file_path}" return nil end end @@ -180,7 +186,7 @@ def get_kube_api_server_url if ENV["KUBERNETES_SERVICE_HOST"] && ENV["KUBERNETES_PORT_443_TCP_PORT"] return "https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}" else - $log.warn ("Kubernetes environment variable not set KUBERNETES_SERVICE_HOST: #{ENV["KUBERNETES_SERVICE_HOST"]} KUBERNETES_PORT_443_TCP_PORT: #{ENV["KUBERNETES_PORT_443_TCP_PORT"]}. Unable to form resourceUri") + @log.warn "Kubernetes environment variable not set KUBERNETES_SERVICE_HOST: #{ENV["KUBERNETES_SERVICE_HOST"]} KUBERNETES_PORT_443_TCP_PORT: #{ENV["KUBERNETES_PORT_443_TCP_PORT"]}. Unable to form resourceUri" return nil end end From 564e99a0d533d4ed7a21cc1dbd5b8a92872da9df Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 15 Aug 2020 14:10:07 -0700 Subject: [PATCH 20/24] fix bug on creating crd resource --- source/plugins/ruby/arc_k8s_cluster_identity.rb | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 95136381b..8d2414c46 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -131,19 +131,22 @@ def renew_near_expiry_token() cluster_identity_resource_namespace: @@cluster_identity_resource_namespace, cluster_identity_resource_name: @@cluster_identity_resource_name, } + crd_request_body = get_crd_request_body + crd_request_body_json = crd_request_body.to_json update_request = Net::HTTP::Patch.new(crd_request_uri) update_request["Content-Type"] = "application/merge-patch+json" update_request["Authorization"] = "Bearer #{@service_account_token}" - update_request_body = get_update_request_body - update_request.body = update_request_body.to_json + update_request.body = crd_request_body_json update_response = @http_client.request(update_request) @log.info "Got response of #{update_response.code} for PATCH #{crd_request_uri} @ #{Time.now.utc.iso8601}" if update_response.code.to_i == 404 @log.info "since crd resource doesnt exist since creating crd resource : #{@@cluster_identity_resource_name} @ #{Time.now.utc.iso8601}" - update_request = Net::HTTP::Post.new(crd_request_uri) - update_request["Content-Type"] = "application/json" - update_response = @http_client.request(update_request) - @log.info "Got response of #{update_response.code} for POST #{crd_request_uri} @ #{Time.now.utc.iso8601}" + create_request = Net::HTTP::Post.new(crd_request_uri) + create_request["Content-Type"] = "application/json" + create_request["Authorization"] = "Bearer #{@service_account_token}" + create_request.body = crd_request_body_json + create_response = @http_client.request(create_request) + @log.info "Got response of #{create_response.code} for POST #{crd_request_uri} @ #{Time.now.utc.iso8601}" end rescue => err @log.warn "renew_near_expiry_token call failed: #{err}" @@ -193,7 +196,7 @@ def get_kube_api_server_url private - def get_update_request_body + def get_crd_request_body body = {} body["apiVersion"] = @@cluster_config_crd_api_version body["kind"] = @@cluster_identity_request_kind From 776eca5731d8119500629ec81ed777f4a45e12a3 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 15 Aug 2020 16:26:29 -0700 Subject: [PATCH 21/24] remove update permission since not required --- charts/azuremonitor-containers/templates/omsagent-rbac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml index 876d64a9f..dee141554 100644 --- a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml @@ -30,7 +30,7 @@ rules: - apiGroups: ["clusterconfig.azure.com"] resources: ["azureclusteridentityrequests"] resourceNames: ["container-insights-clusteridentityrequest"] - verbs: ["get", "create", "patch", "update"] + verbs: ["get", "create", "patch"] - nonResourceURLs: ["/metrics"] verbs: ["get"] - apiGroups: [""] From bf933390432b90ce7942e508acec249e6b0e643e Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 19 Aug 2020 15:13:22 -0700 Subject: [PATCH 22/24] fixed some bugs --- README.md | 6 +++++- .../windows/dockerbuild/build-and-publish-docker-image.ps1 | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 659fe0161..d5d874c9c 100644 --- a/README.md +++ b/README.md @@ -200,11 +200,15 @@ docker build -t /: --build-arg IMAGE_TAG= . docker push /: ``` -### Build Cert generator, Out OMS Plugun and Docker Image and Publish Docker Image +### Build Cert generator, Out OMS Plugin and Docker Image and Publish Docker Image If you have code cloned on to windows, you can built everything for windows agent on windows machine via below instructions ``` +# install pre-requisites if you havent installed already +cd %userprofile%\Docker-Provider\kubernetes\windows # based on your repo path +.\install-build-pre-requisites.ps1 + cd %userprofile%\Docker-Provider\kubernetes\windows\dockerbuild # based on your repo path docker login # if you want to publish the image to acr then login to acr via `docker login ` powershell -ExecutionPolicy bypass # switch to powershell if you are not on powershell already diff --git a/kubernetes/windows/dockerbuild/build-and-publish-docker-image.ps1 b/kubernetes/windows/dockerbuild/build-and-publish-docker-image.ps1 index 27be90d48..dbcfa6097 100644 --- a/kubernetes/windows/dockerbuild/build-and-publish-docker-image.ps1 +++ b/kubernetes/windows/dockerbuild/build-and-publish-docker-image.ps1 @@ -35,7 +35,7 @@ $imagerepo = $imageparts[0] if ($imagetag.StartsWith("win-") -eq $false) { Write-Host "adding win- prefix image tag since its not provided" - $imagetag = "win"-$imagetag + $imagetag = "win-$imagetag" } Write-Host "image tag used is :$imagetag" From 69543ad3734145a46a654a6bd45e80d860373ad1 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 19 Aug 2020 18:00:45 -0700 Subject: [PATCH 23/24] fix pr feedback --- .../templates/omsagent-rbac.yaml | 2 +- source/plugins/ruby/KubernetesApiClient.rb | 17 ++++++- .../plugins/ruby/arc_k8s_cluster_identity.rb | 50 +++++++++++-------- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml index dee141554..8c7158299 100644 --- a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml @@ -36,7 +36,7 @@ rules: - apiGroups: [""] resources: ["secrets"] resourceNames: ["container-insights-clusteridentityrequest-token"] - verbs: ["list", "get", "watch"] + verbs: ["list", "get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/source/plugins/ruby/KubernetesApiClient.rb b/source/plugins/ruby/KubernetesApiClient.rb index 987d290aa..36dcdd8c6 100644 --- a/source/plugins/ruby/KubernetesApiClient.rb +++ b/source/plugins/ruby/KubernetesApiClient.rb @@ -99,7 +99,6 @@ def getResourceUri(resource, api_group) elsif api_group == @@ApiGroupHPA return "https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}/apis/" + @@ApiGroupHPA + "/" + @@ApiVersionHPA + "/" + resource end - else @Log.warn ("Kubernetes environment variable not set KUBERNETES_SERVICE_HOST: #{ENV["KUBERNETES_SERVICE_HOST"]} KUBERNETES_PORT_443_TCP_PORT: #{ENV["KUBERNETES_PORT_443_TCP_PORT"]}. Unable to form resourceUri") return nil @@ -743,7 +742,7 @@ def getResourcesAndContinuationToken(uri, api_group: nil) resourceInventory = nil begin @Log.info "KubernetesApiClient::getResourcesAndContinuationToken : Getting resources from Kube API using url: #{uri} @ #{Time.now.utc.iso8601}" - resourceInfo = getKubeResourceInfo(uri, api_group:api_group) + resourceInfo = getKubeResourceInfo(uri, api_group: api_group) @Log.info "KubernetesApiClient::getResourcesAndContinuationToken : Done getting resources from Kube API using url: #{uri} @ #{Time.now.utc.iso8601}" if !resourceInfo.nil? @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:Start:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" @@ -761,5 +760,19 @@ def getResourcesAndContinuationToken(uri, api_group: nil) end return continuationToken, resourceInventory end #getResourcesAndContinuationToken + + def getKubeAPIServerUrl + apiServerUrl = nil + begin + if ENV["KUBERNETES_SERVICE_HOST"] && ENV["KUBERNETES_PORT_443_TCP_PORT"] + apiServerUrl = "https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}" + else + @Log.warn "Kubernetes environment variable not set KUBERNETES_SERVICE_HOST: #{ENV["KUBERNETES_SERVICE_HOST"]} KUBERNETES_PORT_443_TCP_PORT: #{ENV["KUBERNETES_PORT_443_TCP_PORT"]}. Unable to form resourceUri" + end + rescue => errorStr + @Log.warn "KubernetesApiClient::getKubeAPIServerUrl:Failed #{errorStr}" + end + return apiServerUrl + end end end diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 8d2414c46..ef55c3257 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -6,6 +6,8 @@ require "yajl/json_gem" require "base64" require "time" +require_relative "KubernetesApiClient" +require_relative "ApplicationInsightsUtility" class ArcK8sClusterIdentity # this arc k8s crd version and arc k8s uses corresponding version v1beta1 vs v1 based on the k8s version for apiextensions.k8s.io @@ -26,7 +28,10 @@ def initialize @cached_access_token = String.new @token_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" @cert_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - @kube_api_server_url = get_kube_api_server_url + @kube_api_server_url = KubernetesApiClient.getKubeAPIServerUrl + if @kube_api_server_url.nil? + @log.warn "got api server url nil from KubernetesApiClient.getKubeAPIServerUrl @ #{Time.now.utc.iso8601}" + end @http_client = get_http_client @service_account_token = get_service_account_token @log.info "initialize complete @ #{Time.now.utc.iso8601}" @@ -63,6 +68,7 @@ def get_cluster_identity_token() end rescue => err @log.warn "get_cluster_identity_token failed: #{err}" + ApplicationInsightsUtility.sendExceptionTelemetry(err, { "FeatureArea" => "MDM" }) end return @cached_access_token end @@ -89,6 +95,7 @@ def get_token_from_secret(token_secret_name, token_secret_data_name) end rescue => err @log.warn "get_token_from_secret API call failed: #{err}" + ApplicationInsightsUtility.sendExceptionTelemetry(err, { "FeatureArea" => "MDM" }) end return token end @@ -117,6 +124,7 @@ def get_token_reference_from_crd() end rescue => err @log.warn "get_token_reference_from_crd call failed: #{err}" + ApplicationInsightsUtility.sendExceptionTelemetry(err, { "FeatureArea" => "MDM" }) end return tokenReference end @@ -150,6 +158,7 @@ def renew_near_expiry_token() end rescue => err @log.warn "renew_near_expiry_token call failed: #{err}" + ApplicationInsightsUtility.sendExceptionTelemetry(err, { "FeatureArea" => "MDM" }) end end @@ -164,34 +173,31 @@ def get_service_account_token() @log.warn "Unable to read token string from #{@token_file_path}" return nil end + rescue => err + @log.warn "get_service_account_token call failed: #{err}" + ApplicationInsightsUtility.sendExceptionTelemetry(err, { "FeatureArea" => "MDM" }) end end private def get_http_client() - kube_api_server_url = get_kube_api_server_url - base_api_server_url = URI.parse(kube_api_server_url) - http = Net::HTTP.new(base_api_server_url.host, base_api_server_url.port) - http.use_ssl = true - if !File.exist?(@cert_file_path) - raise "#{@cert_file_path} doesnt exist" - else - http.ca_file = @cert_file_path - end - http.verify_mode = OpenSSL::SSL::VERIFY_PEER - return http - end - - private - - def get_kube_api_server_url - if ENV["KUBERNETES_SERVICE_HOST"] && ENV["KUBERNETES_PORT_443_TCP_PORT"] - return "https://#{ENV["KUBERNETES_SERVICE_HOST"]}:#{ENV["KUBERNETES_PORT_443_TCP_PORT"]}" - else - @log.warn "Kubernetes environment variable not set KUBERNETES_SERVICE_HOST: #{ENV["KUBERNETES_SERVICE_HOST"]} KUBERNETES_PORT_443_TCP_PORT: #{ENV["KUBERNETES_PORT_443_TCP_PORT"]}. Unable to form resourceUri" - return nil + begin + base_api_server_url = URI.parse(@kube_api_server_url) + http = Net::HTTP.new(base_api_server_url.host, base_api_server_url.port) + http.use_ssl = true + if !File.exist?(@cert_file_path) + raise "#{@cert_file_path} doesnt exist" + else + http.ca_file = @cert_file_path + end + http.verify_mode = OpenSSL::SSL::VERIFY_PEER + return http + rescue => err + @log.warn "Unable to create http client #{err}" + ApplicationInsightsUtility.sendExceptionTelemetry(err, { "FeatureArea" => "MDM" }) end + return nil end private From e5b484339d8821223fc7d34d4fb8d704cabbe578 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 19 Aug 2020 18:02:09 -0700 Subject: [PATCH 24/24] remove list since its not required --- charts/azuremonitor-containers/templates/omsagent-rbac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml index 8c7158299..4f7408e7c 100644 --- a/charts/azuremonitor-containers/templates/omsagent-rbac.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-rbac.yaml @@ -36,7 +36,7 @@ rules: - apiGroups: [""] resources: ["secrets"] resourceNames: ["container-insights-clusteridentityrequest-token"] - verbs: ["list", "get"] + verbs: ["get"] --- kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1