From df5e0120638462cab2f96bd9f9d3974ffcb1b394 Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 10 Jan 2020 17:21:22 -0800 Subject: [PATCH 01/17] msi changes --- source/code/plugin/out_mdm.rb | 71 ++++++++++++++++++++++++++--------- 1 file changed, 53 insertions(+), 18 deletions(-) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index 0a4e601b2..aeff7bd3f 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -12,7 +12,7 @@ def initialize require "net/http" require "net/https" require "uri" - require 'yajl/json_gem' + require "yajl/json_gem" require_relative "KubernetesApiClient" require_relative "ApplicationInsightsUtility" @@ -20,11 +20,16 @@ def initialize @@grant_type = "client_credentials" @@azure_json_path = "/etc/kubernetes/host/azure.json" @@post_request_url_template = "https://%{aks_region}.monitoring.azure.com%{aks_resource_id}/metrics" - @@token_url_template = "https://login.microsoftonline.com/%{tenant_id}/oauth2/token" + @@aad_token_url_template = "https://login.microsoftonline.com/%{tenant_id}/oauth2/token" + + # msiEndpoint is the well known endpoint for getting MSI authentications tokens + @@msi_endpoint = "http://169.254.169.254/metadata/identity/oauth2/token" + @@userAssignedClientId = ENV["AzureUserAssignedIdentityClientID"] + @@plugin_name = "AKSCustomMetricsMDM" @data_hash = {} - @token_url = nil + @aad_token_url = nil @http_client = nil @token_expiry_time = Time.now @cached_access_token = String.new @@ -32,6 +37,8 @@ def initialize @first_post_attempt_made = false @can_send_data_to_mdm = true @last_telemetry_sent_time = nil + # Setting useMsi to false by default + @useMsi = false end def configure(conf) @@ -56,12 +63,21 @@ def start @log.info "Environment Variable AKS_REGION is not set.. " @can_send_data_to_mdm = false else - aks_region = aks_region.gsub(" ","") + aks_region = aks_region.gsub(" ", "") end if @can_send_data_to_mdm @log.info "MDM Metrics supported in #{aks_region} region" - @token_url = @@token_url_template % {tenant_id: @data_hash["tenantId"]} + + # Check to see if "useManagedIdentityExtension" is set to true + useManagedIdentityExtension = @data_hash["useManagedIdentityExtension"] + if (!useManagedIdentityExtension.nil? && (useManagedIdentityExtension.casecmp("true") == 0)) + @useMsi = true + else + @useMsi = false + @token_url = @@token_url_template % {tenant_id: @data_hash["tenantId"]} + end + @cached_access_token = get_access_token @@post_request_url = @@post_request_url_template % {aks_region: aks_region, aks_resource_id: aks_resource_id} @post_request_uri = URI.parse(@@post_request_url) @@ -76,27 +92,47 @@ def start @can_send_data_to_mdm = false return end - end # get the access token only if the time to expiry is less than 5 minutes def get_access_token if @cached_access_token.to_s.empty? || (Time.now + 5 * 60 > @token_expiry_time) # token is valid for 60 minutes. Refresh token 5 minutes from expiration @log.info "Refreshing access token for out_mdm plugin.." - token_uri = URI.parse(@token_url) + + if (!!@useMsi) + token_uri = URI.parse(@msi_endpoint) + else + aad_token_url = @@token_url_template % {tenant_id: @data_hash["tenantId"]} + token_uri = URI.parse(aad_token_url) + end + http_access_token = Net::HTTP.new(token_uri.host, token_uri.port) http_access_token.use_ssl = true token_request = Net::HTTP::Post.new(token_uri.request_uri) - token_request.set_form_data( - { - "grant_type" => @@grant_type, - "client_id" => @data_hash["aadClientId"], - "client_secret" => @data_hash["aadClientSecret"], - "resource" => @@token_resource_url, - } - ) + + #TODO:Add nil checks for env vars + if (!!@useMsi && !@@userAssignedClientId.nil?) + token_request = + token_request.set_form_data( + { + "grant_type" => @@grant_type, + "client_id" => @@userAssignedClientId, + "resource" => @@token_resource_url, + } + ) + else + token_request.set_form_data( + { + "grant_type" => @@grant_type, + "client_id" => @data_hash["aadClientId"], + "client_secret" => @data_hash["aadClientSecret"], + "resource" => @@token_resource_url, + } + ) + end token_response = http_access_token.request(token_request) + # Handle the case where the response is not 200 parsed_json = JSON.parse(token_response.body) @token_expiry_time = Time.now + 59 * 60 # set the expiry time to be ~one hour from current time @@ -162,10 +198,9 @@ def send_to_mdm(post_body) response.value # this throws for non 200 HTTP response code @log.info "HTTP Post Response Code : #{response.code}" if @last_telemetry_sent_time.nil? || @last_telemetry_sent_time + 60 * 60 < Time.now - ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMSendSuccessful", {}) - @last_telemetry_sent_time = Time.now + ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMSendSuccessful", {}) + @last_telemetry_sent_time = Time.now end - rescue Net::HTTPServerException => e @log.info "Failed to Post Metrics to MDM : #{e} Response: #{response}" @log.debug_backtrace(e.backtrace) From 9009b8f8ee7602e840a07ddd367ac79acfab202a Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 13 Jan 2020 15:48:15 -0800 Subject: [PATCH 02/17] out mdm changes --- source/code/plugin/out_mdm.rb | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index aeff7bd3f..a70834aa0 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -69,15 +69,27 @@ def start if @can_send_data_to_mdm @log.info "MDM Metrics supported in #{aks_region} region" - # Check to see if "useManagedIdentityExtension" is set to true - useManagedIdentityExtension = @data_hash["useManagedIdentityExtension"] - if (!useManagedIdentityExtension.nil? && (useManagedIdentityExtension.casecmp("true") == 0)) - @useMsi = true - else + # Check to see if SP exists, if it does use SP. Else, use msi + sp_client_id = @data_hash["aadClientId"] + sp_client_secret = @data_hash["aadClientSecret"] + + if (!sp_client_id.nil? && !sp_client_id.empty? && !sp_client_secret.nil? && !sp_client_secret.empty?) @useMsi = false @token_url = @@token_url_template % {tenant_id: @data_hash["tenantId"]} + else + @useMsi = true end + #Commenting this out in case we need to check msi first + # Check to see if "useManagedIdentityExtension" is set to true + # useManagedIdentityExtension = @data_hash["useManagedIdentityExtension"] + # if (!useManagedIdentityExtension.nil? && (useManagedIdentityExtension.casecmp("true") == 0)) + # @useMsi = true + # else + # @useMsi = false + # @token_url = @@token_url_template % {tenant_id: @data_hash["tenantId"]} + # end + @cached_access_token = get_access_token @@post_request_url = @@post_request_url_template % {aks_region: aks_region, aks_resource_id: aks_resource_id} @post_request_uri = URI.parse(@@post_request_url) @@ -111,7 +123,7 @@ def get_access_token token_request = Net::HTTP::Post.new(token_uri.request_uri) #TODO:Add nil checks for env vars - if (!!@useMsi && !@@userAssignedClientId.nil?) + if (!!@useMsi && @@userAssignedClientId.nil?) token_request = token_request.set_form_data( { From 8f3f3524c08ae0e0d45e59b2ded1cc396523edb7 Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 13 Jan 2020 16:11:52 -0800 Subject: [PATCH 03/17] changes complete --- source/code/plugin/out_mdm.rb | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index a70834aa0..ff33bd975 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -29,7 +29,9 @@ def initialize @@plugin_name = "AKSCustomMetricsMDM" @data_hash = {} - @aad_token_url = nil + #@aad_token_url = nil + #@token_url = nil + @parsed_token_uri = nil @http_client = nil @token_expiry_time = Time.now @cached_access_token = String.new @@ -75,9 +77,11 @@ def start if (!sp_client_id.nil? && !sp_client_id.empty? && !sp_client_secret.nil? && !sp_client_secret.empty?) @useMsi = false - @token_url = @@token_url_template % {tenant_id: @data_hash["tenantId"]} + aad_token_url = @@token_url_template % {tenant_id: @data_hash["tenantId"]} + @parsed_token_uri = URI.parse(aad_token_url) else @useMsi = true + @parsed_token_uri = URI.parse(@msi_endpoint) end #Commenting this out in case we need to check msi first @@ -111,24 +115,26 @@ def get_access_token if @cached_access_token.to_s.empty? || (Time.now + 5 * 60 > @token_expiry_time) # token is valid for 60 minutes. Refresh token 5 minutes from expiration @log.info "Refreshing access token for out_mdm plugin.." - if (!!@useMsi) - token_uri = URI.parse(@msi_endpoint) - else - aad_token_url = @@token_url_template % {tenant_id: @data_hash["tenantId"]} - token_uri = URI.parse(aad_token_url) - end + # if (!!@useMsi) + # token_uri = URI.parse(@msi_endpoint) + # else + # aad_token_url = @@token_url_template % {tenant_id: @data_hash["tenantId"]} + # token_uri = URI.parse(aad_token_url) + # end - http_access_token = Net::HTTP.new(token_uri.host, token_uri.port) + http_access_token = Net::HTTP.new(@parsed_token_uri.host, @parsed_token_uri.port) http_access_token.use_ssl = true token_request = Net::HTTP::Post.new(token_uri.request_uri) - #TODO:Add nil checks for env vars - if (!!@useMsi && @@userAssignedClientId.nil?) + if (!!@useMsi) + if(@@userAssignedClientId.nil?) + @log.info "User assigned client id is nil, making request with empty client id" + end token_request = token_request.set_form_data( { "grant_type" => @@grant_type, - "client_id" => @@userAssignedClientId, + "client_id" => (@@userAssignedClientId.nil?)? "" : @@userAssignedClientId, "resource" => @@token_resource_url, } ) From ccd8f69ee285bcc16d994fb20b8e7dba503bd6f4 Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 13 Jan 2020 16:14:59 -0800 Subject: [PATCH 04/17] bug fixes --- source/code/plugin/out_mdm.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index ff33bd975..878dc5867 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -77,7 +77,7 @@ def start if (!sp_client_id.nil? && !sp_client_id.empty? && !sp_client_secret.nil? && !sp_client_secret.empty?) @useMsi = false - aad_token_url = @@token_url_template % {tenant_id: @data_hash["tenantId"]} + aad_token_url = @@aad_token_url_template % {tenant_id: @data_hash["tenantId"]} @parsed_token_uri = URI.parse(aad_token_url) else @useMsi = true @@ -124,7 +124,7 @@ def get_access_token http_access_token = Net::HTTP.new(@parsed_token_uri.host, @parsed_token_uri.port) http_access_token.use_ssl = true - token_request = Net::HTTP::Post.new(token_uri.request_uri) + token_request = Net::HTTP::Post.new(@parsed_token_uri.request_uri) if (!!@useMsi) if(@@userAssignedClientId.nil?) From 8a8b69f463b28ac6a5dff3cbccfb42178ffd8a76 Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 13 Jan 2020 16:17:16 -0800 Subject: [PATCH 05/17] adding comments --- source/code/plugin/out_mdm.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index 878dc5867..dd857077a 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -127,6 +127,7 @@ def get_access_token token_request = Net::HTTP::Post.new(@parsed_token_uri.request_uri) if (!!@useMsi) + @log.info "Using msi to get the token to post MDM data" if(@@userAssignedClientId.nil?) @log.info "User assigned client id is nil, making request with empty client id" end @@ -139,6 +140,7 @@ def get_access_token } ) else + @log.info "Using SP to get the token to post MDM data" token_request.set_form_data( { "grant_type" => @@grant_type, From 544665b3e19722678f8c6ec97d32ef534c089421 Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 13 Jan 2020 20:18:15 -0800 Subject: [PATCH 06/17] changes --- source/code/plugin/out_mdm.rb | 112 +++++++++++++++++----------------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index dd857077a..a0402a176 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -23,8 +23,9 @@ def initialize @@aad_token_url_template = "https://login.microsoftonline.com/%{tenant_id}/oauth2/token" # msiEndpoint is the well known endpoint for getting MSI authentications tokens - @@msi_endpoint = "http://169.254.169.254/metadata/identity/oauth2/token" - @@userAssignedClientId = ENV["AzureUserAssignedIdentityClientID"] + @@msi_endpoint_template = "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&client_id=%{user_assigned_client_id}&resource=%{resource}" + # @@userAssignedClientId = ENV["AzureUserAssignedIdentityClientID"] + @@userAssignedClientId = "8bc13436-46d6-4a1c-b75e-92d676001d23" @@plugin_name = "AKSCustomMetricsMDM" @@ -75,26 +76,35 @@ def start sp_client_id = @data_hash["aadClientId"] sp_client_secret = @data_hash["aadClientSecret"] - if (!sp_client_id.nil? && !sp_client_id.empty? && !sp_client_secret.nil? && !sp_client_secret.empty?) + #Uncomment this if you want to check sp first + # if (!sp_client_id.nil? && !sp_client_id.empty? && !sp_client_secret.nil? && !sp_client_secret.empty?) + # @useMsi = false + # aad_token_url = @@aad_token_url_template % {tenant_id: @data_hash["tenantId"]} + # @parsed_token_uri = URI.parse(aad_token_url) + # else + # @useMsi = true + # @parsed_token_uri = URI.parse(@msi_endpoint) + # end + + # Check to see if "useManagedIdentityExtension" is set to true + useManagedIdentityExtension = @data_hash["useManagedIdentityExtension"] + if (!useManagedIdentityExtension.nil? && useManagedIdentityExtension == true) + if !@@userAssignedClientId.nil? && !@@userAssignedClientId.empty? + @useMsi = true + msi_endpoint = @@msi_endpoint_template % {user_assigned_client_id: @@userAssignedClientId, resource: @@token_resource_url} + @parsed_token_uri = URI.parse(msi_endpoint) + @cached_access_token = get_access_token + else + @can_send_data_to_mdm = false + @log.info "User assigned client id is nil or empty, cannot post data to MDM using MSI" + end + else @useMsi = false aad_token_url = @@aad_token_url_template % {tenant_id: @data_hash["tenantId"]} @parsed_token_uri = URI.parse(aad_token_url) - else - @useMsi = true - @parsed_token_uri = URI.parse(@msi_endpoint) + @cached_access_token = get_access_token end - #Commenting this out in case we need to check msi first - # Check to see if "useManagedIdentityExtension" is set to true - # useManagedIdentityExtension = @data_hash["useManagedIdentityExtension"] - # if (!useManagedIdentityExtension.nil? && (useManagedIdentityExtension.casecmp("true") == 0)) - # @useMsi = true - # else - # @useMsi = false - # @token_url = @@token_url_template % {tenant_id: @data_hash["tenantId"]} - # end - - @cached_access_token = get_access_token @@post_request_url = @@post_request_url_template % {aks_region: aks_region, aks_resource_id: aks_resource_id} @post_request_uri = URI.parse(@@post_request_url) @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port) @@ -112,53 +122,43 @@ def start # get the access token only if the time to expiry is less than 5 minutes def get_access_token - if @cached_access_token.to_s.empty? || (Time.now + 5 * 60 > @token_expiry_time) # token is valid for 60 minutes. Refresh token 5 minutes from expiration - @log.info "Refreshing access token for out_mdm plugin.." - - # if (!!@useMsi) - # token_uri = URI.parse(@msi_endpoint) - # else - # aad_token_url = @@token_url_template % {tenant_id: @data_hash["tenantId"]} - # token_uri = URI.parse(aad_token_url) - # end - - http_access_token = Net::HTTP.new(@parsed_token_uri.host, @parsed_token_uri.port) - http_access_token.use_ssl = true - token_request = Net::HTTP::Post.new(@parsed_token_uri.request_uri) - - if (!!@useMsi) - @log.info "Using msi to get the token to post MDM data" - if(@@userAssignedClientId.nil?) - @log.info "User assigned client id is nil, making request with empty client id" - end - token_request = + begin + if @cached_access_token.to_s.empty? || (Time.now + 5 * 60 > @token_expiry_time) # token is valid for 60 minutes. Refresh token 5 minutes from expiration + @log.info "Refreshing access token for out_mdm plugin.." + + http_access_token = Net::HTTP.new(@parsed_token_uri.host, @parsed_token_uri.port) + + if (!!@useMsi) + @log.info "Using msi to get the token to post MDM data" + http_access_token.use_ssl = false + token_request = Net::HTTP::Get.new(@parsed_token_uri.request_uri) + token_request["Metadata"] = true + + else + @log.info "Using SP to get the token to post MDM data" + http_access_token.use_ssl = true + token_request = Net::HTTP::Post.new(@parsed_token_uri.request_uri) token_request.set_form_data( { "grant_type" => @@grant_type, - "client_id" => (@@userAssignedClientId.nil?)? "" : @@userAssignedClientId, + "client_id" => @data_hash["aadClientId"], + "client_secret" => @data_hash["aadClientSecret"], "resource" => @@token_resource_url, } ) - else - @log.info "Using SP to get the token to post MDM data" - token_request.set_form_data( - { - "grant_type" => @@grant_type, - "client_id" => @data_hash["aadClientId"], - "client_secret" => @data_hash["aadClientSecret"], - "resource" => @@token_resource_url, - } - ) - end - - token_response = http_access_token.request(token_request) + end - # Handle the case where the response is not 200 - parsed_json = JSON.parse(token_response.body) - @token_expiry_time = Time.now + 59 * 60 # set the expiry time to be ~one hour from current time - @cached_access_token = parsed_json["access_token"] + @log.info "making request.." + token_response = http_access_token.request(token_request) + # Handle the case where the response is not 200 + parsed_json = JSON.parse(token_response.body) + @token_expiry_time = Time.now + 59 * 60 # set the expiry time to be ~one hour from current time + @cached_access_token = parsed_json["access_token"] + end + @cached_access_token + rescue => err + @log.info "Exception in get_access_token: #{err}" end - @cached_access_token end def write_status_file(success, message) From 29c27a347e3c4a69de373b9475e40f6c5569ad8f Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 14 Jan 2020 19:01:26 -0800 Subject: [PATCH 07/17] changes --- source/code/plugin/out_mdm.rb | 63 ++++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 26 deletions(-) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index a0402a176..81d648fef 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -24,8 +24,8 @@ def initialize # msiEndpoint is the well known endpoint for getting MSI authentications tokens @@msi_endpoint_template = "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&client_id=%{user_assigned_client_id}&resource=%{resource}" - # @@userAssignedClientId = ENV["AzureUserAssignedIdentityClientID"] - @@userAssignedClientId = "8bc13436-46d6-4a1c-b75e-92d676001d23" + @@userAssignedClientId = ENV["USER_ASSIGNED_IDENTITY_CLIENT_ID"] + # @@userAssignedClientId = "8bc13436-46d6-4a1c-b75e-92d676001d23" @@plugin_name = "AKSCustomMetricsMDM" @@ -77,34 +77,44 @@ def start sp_client_secret = @data_hash["aadClientSecret"] #Uncomment this if you want to check sp first - # if (!sp_client_id.nil? && !sp_client_id.empty? && !sp_client_secret.nil? && !sp_client_secret.empty?) - # @useMsi = false - # aad_token_url = @@aad_token_url_template % {tenant_id: @data_hash["tenantId"]} - # @parsed_token_uri = URI.parse(aad_token_url) - # else - # @useMsi = true - # @parsed_token_uri = URI.parse(@msi_endpoint) - # end - - # Check to see if "useManagedIdentityExtension" is set to true - useManagedIdentityExtension = @data_hash["useManagedIdentityExtension"] - if (!useManagedIdentityExtension.nil? && useManagedIdentityExtension == true) - if !@@userAssignedClientId.nil? && !@@userAssignedClientId.empty? - @useMsi = true - msi_endpoint = @@msi_endpoint_template % {user_assigned_client_id: @@userAssignedClientId, resource: @@token_resource_url} - @parsed_token_uri = URI.parse(msi_endpoint) - @cached_access_token = get_access_token - else - @can_send_data_to_mdm = false - @log.info "User assigned client id is nil or empty, cannot post data to MDM using MSI" - end - else + if (!sp_client_id.nil? && !sp_client_id.empty?) @useMsi = false aad_token_url = @@aad_token_url_template % {tenant_id: @data_hash["tenantId"]} @parsed_token_uri = URI.parse(aad_token_url) @cached_access_token = get_access_token + else + # if !@@userAssignedClientId.nil? && !@@userAssignedClientId.empty? + @useMsi = true + msi_endpoint = @@msi_endpoint_template % {user_assigned_client_id: @@userAssignedClientId, resource: @@token_resource_url} + @parsed_token_uri = URI.parse(msi_endpoint) + @cached_access_token = get_access_token + # else + # @can_send_data_to_mdm = false + # @log.info "User assigned client id is nil or empty, cannot post data to MDM using MSI" + # end end + @cached_access_token = get_access_token + + # Check to see if "useManagedIdentityExtension" is set to true + # useManagedIdentityExtension = @data_hash["useManagedIdentityExtension"] + # if (!useManagedIdentityExtension.nil? && useManagedIdentityExtension == true) + # if !@@userAssignedClientId.nil? && !@@userAssignedClientId.empty? + # @useMsi = true + # msi_endpoint = @@msi_endpoint_template % {user_assigned_client_id: @@userAssignedClientId, resource: @@token_resource_url} + # @parsed_token_uri = URI.parse(msi_endpoint) + # @cached_access_token = get_access_token + # else + # @can_send_data_to_mdm = false + # @log.info "User assigned client id is nil or empty, cannot post data to MDM using MSI" + # end + # else + # @useMsi = false + # aad_token_url = @@aad_token_url_template % {tenant_id: @data_hash["tenantId"]} + # @parsed_token_uri = URI.parse(aad_token_url) + # @cached_access_token = get_access_token + # end + @@post_request_url = @@post_request_url_template % {aks_region: aks_region, aks_resource_id: aks_resource_id} @post_request_uri = URI.parse(@@post_request_url) @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port) @@ -130,12 +140,13 @@ def get_access_token if (!!@useMsi) @log.info "Using msi to get the token to post MDM data" + ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMToken-MSI", {}) http_access_token.use_ssl = false token_request = Net::HTTP::Get.new(@parsed_token_uri.request_uri) token_request["Metadata"] = true - else @log.info "Using SP to get the token to post MDM data" + ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMToken-SP", {}) http_access_token.use_ssl = true token_request = Net::HTTP::Post.new(@parsed_token_uri.request_uri) token_request.set_form_data( @@ -148,7 +159,7 @@ def get_access_token ) end - @log.info "making request.." + @log.info "making request to get token.." token_response = http_access_token.request(token_request) # Handle the case where the response is not 200 parsed_json = JSON.parse(token_response.body) From f87e962bef3edaf4ec310af67ce4e974b5a04edd Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 21 Jan 2020 17:14:00 -0800 Subject: [PATCH 08/17] adding backoff and retry --- source/code/plugin/out_mdm.rb | 103 +++++++++++++++++++++------------- 1 file changed, 63 insertions(+), 40 deletions(-) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index 81d648fef..cebaf3532 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -42,6 +42,8 @@ def initialize @last_telemetry_sent_time = nil # Setting useMsi to false by default @useMsi = false + + @get_access_token_backoff_expiry = Time.now end def configure(conf) @@ -84,10 +86,10 @@ def start @cached_access_token = get_access_token else # if !@@userAssignedClientId.nil? && !@@userAssignedClientId.empty? - @useMsi = true - msi_endpoint = @@msi_endpoint_template % {user_assigned_client_id: @@userAssignedClientId, resource: @@token_resource_url} - @parsed_token_uri = URI.parse(msi_endpoint) - @cached_access_token = get_access_token + @useMsi = true + msi_endpoint = @@msi_endpoint_template % {user_assigned_client_id: @@userAssignedClientId, resource: @@token_resource_url} + @parsed_token_uri = URI.parse(msi_endpoint) + @cached_access_token = get_access_token # else # @can_send_data_to_mdm = false # @log.info "User assigned client id is nil or empty, cannot post data to MDM using MSI" @@ -125,50 +127,71 @@ def start rescue => e @log.info "exception when initializing out_mdm #{e}" ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "MDM"}) - @can_send_data_to_mdm = false + # @can_send_data_to_mdm = false + @get_access_token_backoff_expiry = Time.now + 59 * 60 return end end - # get the access token only if the time to expiry is less than 5 minutes + # get the access token only if the time to expiry is less than 5 minutes and get_access_token_backoff has expired def get_access_token - begin - if @cached_access_token.to_s.empty? || (Time.now + 5 * 60 > @token_expiry_time) # token is valid for 60 minutes. Refresh token 5 minutes from expiration - @log.info "Refreshing access token for out_mdm plugin.." - - http_access_token = Net::HTTP.new(@parsed_token_uri.host, @parsed_token_uri.port) - - if (!!@useMsi) - @log.info "Using msi to get the token to post MDM data" - ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMToken-MSI", {}) - http_access_token.use_ssl = false - token_request = Net::HTTP::Get.new(@parsed_token_uri.request_uri) - token_request["Metadata"] = true + if (Time.now > @get_access_token_backoff_expiry) + http_access_token = nil + retries = 0 + begin + if @cached_access_token.to_s.empty? || (Time.now + 5 * 60 > @token_expiry_time) # token is valid for 60 minutes. Refresh token 5 minutes from expiration + @log.info "Refreshing access token for out_mdm plugin.." + + # http_access_token = Net::HTTP.new(@parsed_token_uri.host, @parsed_token_uri.port) + if (!!@useMsi) + @log.info "Using msi to get the token to post MDM data" + ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMToken-MSI", {}) + @log.info "Opening TCP connection" + http_access_token = Net::HTTP.start(@parsed_token_uri.host, @parsed_token_uri.port, :use_ssl => false) + # http_access_token.use_ssl = false + token_request = Net::HTTP::Get.new(@parsed_token_uri.request_uri) + token_request["Metadata"] = true + else + @log.info "Using SP to get the token to post MDM data" + ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMToken-SP", {}) + @log.info "Opening TCP connection" + http_access_token = Net::HTTP.start(@parsed_token_uri.host, @parsed_token_uri.port, :use_ssl => true) + # http_access_token.use_ssl = true + token_request = Net::HTTP::Post.new(@parsed_token_uri.request_uri) + token_request.set_form_data( + { + "grant_type" => @@grant_type, + "client_id" => @data_hash["aadClientId"], + "client_secret" => @data_hash["aadClientSecret"], + "resource" => @@token_resource_url, + } + ) + end + + @log.info "making request to get token.." + token_response = http_access_token.request(token_request) + # Handle the case where the response is not 200 + parsed_json = JSON.parse(token_response.body) + @token_expiry_time = Time.now + 59 * 60 # set the expiry time to be ~one hour from current time + @cached_access_token = parsed_json["access_token"] + end + @cached_access_token + rescue => err + @log.info "Exception in get_access_token: #{err}" + if (retries < 2) + retries += 1 + @log.info "Retrying request to get token - retry number: #{retries}" + sleep(retries) + retry else - @log.info "Using SP to get the token to post MDM data" - ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMToken-SP", {}) - http_access_token.use_ssl = true - token_request = Net::HTTP::Post.new(@parsed_token_uri.request_uri) - token_request.set_form_data( - { - "grant_type" => @@grant_type, - "client_id" => @data_hash["aadClientId"], - "client_secret" => @data_hash["aadClientSecret"], - "resource" => @@token_resource_url, - } - ) + raise + end + ensure + if http_access_token + @log.info "Closing http connection" + http_access_token.finish end - - @log.info "making request to get token.." - token_response = http_access_token.request(token_request) - # Handle the case where the response is not 200 - parsed_json = JSON.parse(token_response.body) - @token_expiry_time = Time.now + 59 * 60 # set the expiry time to be ~one hour from current time - @cached_access_token = parsed_json["access_token"] end - @cached_access_token - rescue => err - @log.info "Exception in get_access_token: #{err}" end end From befde559ba84a78f366118decbf8beb17c213904 Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 22 Jan 2020 14:39:41 -0800 Subject: [PATCH 09/17] changes for backoff and retry --- source/code/plugin/out_mdm.rb | 50 ++++++++--------------------------- 1 file changed, 11 insertions(+), 39 deletions(-) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index cebaf3532..d950453e5 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -25,7 +25,6 @@ def initialize # msiEndpoint is the well known endpoint for getting MSI authentications tokens @@msi_endpoint_template = "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&client_id=%{user_assigned_client_id}&resource=%{resource}" @@userAssignedClientId = ENV["USER_ASSIGNED_IDENTITY_CLIENT_ID"] - # @@userAssignedClientId = "8bc13436-46d6-4a1c-b75e-92d676001d23" @@plugin_name = "AKSCustomMetricsMDM" @@ -74,61 +73,32 @@ def start if @can_send_data_to_mdm @log.info "MDM Metrics supported in #{aks_region} region" + @@post_request_url = @@post_request_url_template % {aks_region: aks_region, aks_resource_id: aks_resource_id} + @post_request_uri = URI.parse(@@post_request_url) + @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port) + @http_client.use_ssl = true + @log.info "POST Request url: #{@@post_request_url}" + ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMPluginStart", {}) + # Check to see if SP exists, if it does use SP. Else, use msi sp_client_id = @data_hash["aadClientId"] sp_client_secret = @data_hash["aadClientSecret"] - #Uncomment this if you want to check sp first if (!sp_client_id.nil? && !sp_client_id.empty?) @useMsi = false aad_token_url = @@aad_token_url_template % {tenant_id: @data_hash["tenantId"]} @parsed_token_uri = URI.parse(aad_token_url) - @cached_access_token = get_access_token else - # if !@@userAssignedClientId.nil? && !@@userAssignedClientId.empty? @useMsi = true msi_endpoint = @@msi_endpoint_template % {user_assigned_client_id: @@userAssignedClientId, resource: @@token_resource_url} @parsed_token_uri = URI.parse(msi_endpoint) - @cached_access_token = get_access_token - # else - # @can_send_data_to_mdm = false - # @log.info "User assigned client id is nil or empty, cannot post data to MDM using MSI" - # end end @cached_access_token = get_access_token - - # Check to see if "useManagedIdentityExtension" is set to true - # useManagedIdentityExtension = @data_hash["useManagedIdentityExtension"] - # if (!useManagedIdentityExtension.nil? && useManagedIdentityExtension == true) - # if !@@userAssignedClientId.nil? && !@@userAssignedClientId.empty? - # @useMsi = true - # msi_endpoint = @@msi_endpoint_template % {user_assigned_client_id: @@userAssignedClientId, resource: @@token_resource_url} - # @parsed_token_uri = URI.parse(msi_endpoint) - # @cached_access_token = get_access_token - # else - # @can_send_data_to_mdm = false - # @log.info "User assigned client id is nil or empty, cannot post data to MDM using MSI" - # end - # else - # @useMsi = false - # aad_token_url = @@aad_token_url_template % {tenant_id: @data_hash["tenantId"]} - # @parsed_token_uri = URI.parse(aad_token_url) - # @cached_access_token = get_access_token - # end - - @@post_request_url = @@post_request_url_template % {aks_region: aks_region, aks_resource_id: aks_resource_id} - @post_request_uri = URI.parse(@@post_request_url) - @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port) - @http_client.use_ssl = true - @log.info "POST Request url: #{@@post_request_url}" - ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMPluginStart", {}) end rescue => e @log.info "exception when initializing out_mdm #{e}" ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "MDM"}) - # @can_send_data_to_mdm = false - @get_access_token_backoff_expiry = Time.now + 59 * 60 return end end @@ -142,7 +112,6 @@ def get_access_token if @cached_access_token.to_s.empty? || (Time.now + 5 * 60 > @token_expiry_time) # token is valid for 60 minutes. Refresh token 5 minutes from expiration @log.info "Refreshing access token for out_mdm plugin.." - # http_access_token = Net::HTTP.new(@parsed_token_uri.host, @parsed_token_uri.port) if (!!@useMsi) @log.info "Using msi to get the token to post MDM data" ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMToken-MSI", {}) @@ -174,6 +143,7 @@ def get_access_token parsed_json = JSON.parse(token_response.body) @token_expiry_time = Time.now + 59 * 60 # set the expiry time to be ~one hour from current time @cached_access_token = parsed_json["access_token"] + @log.info "Successfully got access token" end @cached_access_token rescue => err @@ -184,7 +154,9 @@ def get_access_token sleep(retries) retry else - raise + @get_access_token_backoff_expiry = Time.now + 59 * 60 + @log.info "@get_access_token_backoff_expiry set to #{@get_access_token_backoff_expiry}" + ApplicationInsightsUtility.sendExceptionTelemetry(err, {"FeatureArea" => "MDM"}) end ensure if http_access_token From 7f6930f9f12586b98cfe1a9d96e3b68b481e2098 Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 22 Jan 2020 18:04:36 -0800 Subject: [PATCH 10/17] moving access token return --- source/code/plugin/out_mdm.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index d950453e5..a20fc02b4 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -145,7 +145,6 @@ def get_access_token @cached_access_token = parsed_json["access_token"] @log.info "Successfully got access token" end - @cached_access_token rescue => err @log.info "Exception in get_access_token: #{err}" if (retries < 2) @@ -165,6 +164,7 @@ def get_access_token end end end + @cached_access_token end def write_status_file(success, message) From 19abbadc9f8fbf7de0f1df2dc6dc9da3e629effd Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 11 Feb 2020 15:57:43 -0800 Subject: [PATCH 11/17] msi string and access toekn refresh time changes --- installer/conf/container.conf | 2 +- source/code/plugin/out_mdm.rb | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/installer/conf/container.conf b/installer/conf/container.conf index 93c250fbb..0e088e7f7 100644 --- a/installer/conf/container.conf +++ b/installer/conf/container.conf @@ -110,5 +110,5 @@ retry_limit 10 retry_wait 5s max_retry_wait 5m - retry_mdm_post_wait_minutes 60 + retry_mdm_post_wait_minutes 30 diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index a20fc02b4..9234c6ce0 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -84,7 +84,7 @@ def start sp_client_id = @data_hash["aadClientId"] sp_client_secret = @data_hash["aadClientSecret"] - if (!sp_client_id.nil? && !sp_client_id.empty?) + if (!sp_client_id.nil? && !sp_client_id.empty? && sp_client_id != "msi") @useMsi = false aad_token_url = @@aad_token_url_template % {tenant_id: @data_hash["tenantId"]} @parsed_token_uri = URI.parse(aad_token_url) @@ -109,7 +109,7 @@ def get_access_token http_access_token = nil retries = 0 begin - if @cached_access_token.to_s.empty? || (Time.now + 5 * 60 > @token_expiry_time) # token is valid for 60 minutes. Refresh token 5 minutes from expiration + if @cached_access_token.to_s.empty? || (Time.now + 5 * 60 > @token_expiry_time) # Refresh token 5 minutes from expiration @log.info "Refreshing access token for out_mdm plugin.." if (!!@useMsi) @@ -141,7 +141,7 @@ def get_access_token token_response = http_access_token.request(token_request) # Handle the case where the response is not 200 parsed_json = JSON.parse(token_response.body) - @token_expiry_time = Time.now + 59 * 60 # set the expiry time to be ~one hour from current time + @token_expiry_time = Time.now + 29 * 60 # set the expiry time to be ~thirty minutes from current time @cached_access_token = parsed_json["access_token"] @log.info "Successfully got access token" end @@ -153,7 +153,7 @@ def get_access_token sleep(retries) retry else - @get_access_token_backoff_expiry = Time.now + 59 * 60 + @get_access_token_backoff_expiry = Time.now + 29 * 60 @log.info "@get_access_token_backoff_expiry set to #{@get_access_token_backoff_expiry}" ApplicationInsightsUtility.sendExceptionTelemetry(err, {"FeatureArea" => "MDM"}) end From de3e868e734173e452fa96c862e6abf13d520b44 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 11 Feb 2020 16:00:57 -0800 Subject: [PATCH 12/17] removing comments --- source/code/plugin/out_mdm.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index d48458056..ab3c2899a 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -30,8 +30,6 @@ def initialize @@record_batch_size = 2600 @data_hash = {} - #@aad_token_url = nil - #@token_url = nil @parsed_token_uri = nil @http_client = nil @token_expiry_time = Time.now From 8e5916c269bc3549ee8cd040b8e5ef7ee0b025d5 Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 13 Feb 2020 14:30:32 -0800 Subject: [PATCH 13/17] fix windows image collection bug --- source/code/plugin/in_kube_podinventory.rb | 49 ++++++++++++---------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/source/code/plugin/in_kube_podinventory.rb b/source/code/plugin/in_kube_podinventory.rb index 3a8ad2761..3a78d4c05 100644 --- a/source/code/plugin/in_kube_podinventory.rb +++ b/source/code/plugin/in_kube_podinventory.rb @@ -2,7 +2,6 @@ # frozen_string_literal: true module Fluent - require_relative "podinventory_to_mdm" class Kube_PodInventory_Input < Input @@ -37,7 +36,6 @@ def initialize config_param :tag, :string, :default => "oms.containerinsights.KubePodInventory" config_param :custom_metrics_azure_regions, :string - def configure(conf) super @inventoryToMdmConvertor = Inventory2MdmConvertor.new(@custom_metrics_azure_regions) @@ -149,18 +147,25 @@ def populateWindowsContainerInventoryRecord(container, record, containerEnvVaria containerInventoryRecord["Computer"] = record["Computer"] containerInventoryRecord["ContainerHostname"] = record["Computer"] containerInventoryRecord["ElementName"] = containerName - image = container["image"] - repoInfo = image.split("/") - if !repoInfo.nil? - containerInventoryRecord["Repository"] = repoInfo[0] - if !repoInfo[1].nil? - imageInfo = repoInfo[1].split(":") - if !imageInfo.nil? - containerInventoryRecord["Image"] = imageInfo[0] - containerInventoryRecord["ImageTag"] = imageInfo[1] + + # Find delimiters in the string of format repository/image:imagetag + imageValue = container["image"] + if !imageValue.empty? + slashLocation = imageValue.index("/") + colonLocation = imageValue.index(":") + if !colonLocation.nil? + if slashLocation.nil? + # image:imagetag + containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] + else + # repository/image:imagetag + containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] + containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] end + containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] end end + imageIdInfo = container["imageID"] imageIdSplitInfo = imageIdInfo.split("@") if !imageIdSplitInfo.nil? @@ -273,8 +278,8 @@ def parse_and_emit_records(podInventory, serviceList, continuationToken, batchTi # For ARO v3 cluster, skip the pods scheduled on to master or infra nodes if KubernetesApiClient.isAROV3Cluster && !items["spec"].nil? && !items["spec"]["nodeName"].nil? && - ( items["spec"]["nodeName"].downcase.start_with?("infra-") || - items["spec"]["nodeName"].downcase.start_with?("master-") ) + (items["spec"]["nodeName"].downcase.start_with?("infra-") || + items["spec"]["nodeName"].downcase.start_with?("master-")) next end @@ -491,15 +496,15 @@ def parse_and_emit_records(podInventory, serviceList, continuationToken, batchTi router.emit_stream(@tag, eventStream) if eventStream - if continuationToken.nil? #no more chunks in this batch to be sent, get all pod inventory records to send - @log.info "Sending pod inventory mdm records to out_mdm" - pod_inventory_mdm_records = @inventoryToMdmConvertor.get_pod_inventory_mdm_records(batchTime) - @log.info "pod_inventory_mdm_records.size #{pod_inventory_mdm_records.size}" - mdm_pod_inventory_es = MultiEventStream.new - pod_inventory_mdm_records.each {|pod_inventory_mdm_record| - mdm_pod_inventory_es.add(batchTime, pod_inventory_mdm_record) if pod_inventory_mdm_record - } if pod_inventory_mdm_records - router.emit_stream(@@MDMKubePodInventoryTag, mdm_pod_inventory_es) if mdm_pod_inventory_es + if continuationToken.nil? #no more chunks in this batch to be sent, get all pod inventory records to send + @log.info "Sending pod inventory mdm records to out_mdm" + pod_inventory_mdm_records = @inventoryToMdmConvertor.get_pod_inventory_mdm_records(batchTime) + @log.info "pod_inventory_mdm_records.size #{pod_inventory_mdm_records.size}" + mdm_pod_inventory_es = MultiEventStream.new + pod_inventory_mdm_records.each { |pod_inventory_mdm_record| + mdm_pod_inventory_es.add(batchTime, pod_inventory_mdm_record) if pod_inventory_mdm_record + } if pod_inventory_mdm_records + router.emit_stream(@@MDMKubePodInventoryTag, mdm_pod_inventory_es) if mdm_pod_inventory_es end #:optimize:kubeperf merge From 1a4e639abd6db9de9bb05871139cca4e3049a86b Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 19 Feb 2020 11:53:08 -0800 Subject: [PATCH 14/17] adding event count telemetry --- source/code/plugin/in_kube_events.rb | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/source/code/plugin/in_kube_events.rb b/source/code/plugin/in_kube_events.rb index b405afde9..8e586aa36 100644 --- a/source/code/plugin/in_kube_events.rb +++ b/source/code/plugin/in_kube_events.rb @@ -4,7 +4,6 @@ module Fluent class Kube_Event_Input < Input Plugin.register_input("kubeevents", self) - @@KubeEventsStateFile = "/var/opt/microsoft/docker-cimprov/state/KubeEventQueryState.yaml" def initialize @@ -20,6 +19,9 @@ def initialize # 30000 events account to approximately 5MB @EVENTS_CHUNK_SIZE = 30000 + + # Initializing events count for telemetry + @eventsCount = 0 end config_param :run_interval, :time, :default => 60 @@ -55,6 +57,7 @@ def enumerate batchTime = currentTime.utc.iso8601 eventQueryState = getEventQueryState newEventQueryState = [] + @eventsCount = 0 # Initializing continuation token to nil continuationToken = nil @@ -80,6 +83,13 @@ def enumerate # Setting this to nil so that we dont hold memory until GC kicks in eventList = nil writeEventQueryState(newEventQueryState) + + # Flush AppInsights telemetry once all the processing is done, only if the number of events flushed is greater than 0 + if (@eventsCount > 0) + telemetryProperties = {} + ApplicationInsightsUtility.sendMetricTelemetry("EventCount", @eventsCount, {}) + end + rescue => errorStr $log.warn "in_kube_events::enumerate:Failed in enumerate: #{errorStr}" $log.debug_backtrace(errorStr.backtrace) @@ -105,7 +115,7 @@ def parse_and_emit_records(events, eventQueryState, newEventQueryState, batchTim nodeName = items["source"].key?("host") ? items["source"]["host"] : (OMS::Common.get_hostname) # For ARO v3 cluster, drop the master and infra node sourced events to ingest if KubernetesApiClient.isAROV3Cluster && !nodeName.nil? && !nodeName.empty? && - ( nodeName.downcase.start_with?("infra-") || nodeName.downcase.start_with?("master-") ) + (nodeName.downcase.start_with?("infra-") || nodeName.downcase.start_with?("master-")) next end @@ -129,6 +139,7 @@ def parse_and_emit_records(events, eventQueryState, newEventQueryState, batchTim "DataItems" => [record.each { |k, v| record[k] = v }], } eventStream.add(emitTime, wrapper) if wrapper + @eventsCount += 1 end router.emit_stream(@tag, eventStream) if eventStream rescue => errorStr From 9418fe4e1b1e627c4a8ad5b9129aca3b3db6f2db Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 19 Feb 2020 12:32:32 -0800 Subject: [PATCH 15/17] configure normal kube event collection --- installer/scripts/tomlparser.rb | 12 ++++++++++++ source/code/plugin/in_kube_events.rb | 21 +++++++++++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/installer/scripts/tomlparser.rb b/installer/scripts/tomlparser.rb index ba67d023a..5f2596bca 100644 --- a/installer/scripts/tomlparser.rb +++ b/installer/scripts/tomlparser.rb @@ -16,6 +16,7 @@ @logExclusionRegexPattern = "(^((?!stdout|stderr).)*$)" @excludePath = "*.csv2" #some invalid path @enrichContainerLogs = false +@collectAllKubeEvents = false # Use parser to parse the configmap toml file to a ruby structure def parseConfigMap @@ -128,6 +129,16 @@ def populateSettingValuesFromConfigMap(parsedConfig) rescue => errorStr ConfigParseErrorLogger.logError("Exception while reading config map settings for cluster level container log enrichment - #{errorStr}, using defaults, please check config map for errors") end + + #Get kube events enrichment setting + begin + if !parsedConfig[:log_collection_settings][:collect_all_kube_events].nil? && !parsedConfig[:log_collection_settings][:collect_all_kube_events][:enabled].nil? + @collectAllKubeEvents = parsedConfig[:log_collection_settings][:collect_all_kube_events][:enabled] + puts "config::Using config map setting for kube event collection" + end + rescue => errorStr + ConfigParseErrorLogger.logError("Exception while reading config map settings for kube event collection - #{errorStr}, using defaults, please check config map for errors") + end end end @@ -168,6 +179,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) file.write("export AZMON_CLUSTER_COLLECT_ENV_VAR=#{@collectClusterEnvVariables}\n") file.write("export AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH=#{@excludePath}\n") file.write("export AZMON_CLUSTER_CONTAINER_LOG_ENRICH=#{@enrichContainerLogs}\n") + file.write("export AZMON_CLUSTER_COLLECT_ALL_KUBE_EVENTS=#{@collectAllKubeEvents}\n") # Close file after writing all environment variables file.close puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' " diff --git a/source/code/plugin/in_kube_events.rb b/source/code/plugin/in_kube_events.rb index 8e586aa36..bb0ab6f05 100644 --- a/source/code/plugin/in_kube_events.rb +++ b/source/code/plugin/in_kube_events.rb @@ -22,6 +22,9 @@ def initialize # Initializing events count for telemetry @eventsCount = 0 + + # Initilize enable/disable normal event collection + @collectAllKubeEvents = false end config_param :run_interval, :time, :default => 60 @@ -37,6 +40,16 @@ def start @condition = ConditionVariable.new @mutex = Mutex.new @thread = Thread.new(&method(:run_periodic)) + collectAllKubeEventsSetting = ENV["AZMON_CLUSTER_COLLECT_ALL_KUBE_EVENTS"] + if !collectAllKubeEventsSetting.nil? && !collectAllKubeEventsSetting.empty? + if collectAllKubeEventsSetting.casecmp("false") == 0 + @collectAllKubeEvents = false + $log.warn("Normal kube events collection disabled for cluster") + else + @collectAllKubeEvents = true + $log.warn("Normal kube events collection enabled for cluster") + end + end end end @@ -62,7 +75,11 @@ def enumerate # Initializing continuation token to nil continuationToken = nil $log.info("in_kube_events::enumerate : Getting events from Kube API @ #{Time.now.utc.iso8601}") - continuationToken, eventList = KubernetesApiClient.getResourcesAndContinuationToken("events?fieldSelector=type!=Normal&limit=#{@EVENTS_CHUNK_SIZE}") + if @collectAllKubeEvents + continuationToken, eventList = KubernetesApiClient.getResourcesAndContinuationToken("events?limit=#{@EVENTS_CHUNK_SIZE}") + else + continuationToken, eventList = KubernetesApiClient.getResourcesAndContinuationToken("events?fieldSelector=type!=Normal&limit=#{@EVENTS_CHUNK_SIZE}") + end $log.info("in_kube_events::enumerate : Done getting events from Kube API @ #{Time.now.utc.iso8601}") if (!eventList.nil? && !eventList.empty? && eventList.key?("items") && !eventList["items"].nil? && !eventList["items"].empty?) newEventQueryState = parse_and_emit_records(eventList, eventQueryState, newEventQueryState, batchTime) @@ -87,9 +104,9 @@ def enumerate # Flush AppInsights telemetry once all the processing is done, only if the number of events flushed is greater than 0 if (@eventsCount > 0) telemetryProperties = {} + telemetryProperties["CollectAllKubeEvents"] = @collectAllKubeEvents ApplicationInsightsUtility.sendMetricTelemetry("EventCount", @eventsCount, {}) end - rescue => errorStr $log.warn "in_kube_events::enumerate:Failed in enumerate: #{errorStr}" $log.debug_backtrace(errorStr.backtrace) From fedeaf7a3ecae9292c0dd27b9e8b6f01b82d855c Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 20 Feb 2020 16:43:32 -0800 Subject: [PATCH 16/17] moving it to constant --- source/code/plugin/out_mdm.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index ab3c2899a..2f90b89ee 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -29,6 +29,8 @@ def initialize @@plugin_name = "AKSCustomMetricsMDM" @@record_batch_size = 2600 + @@tokenRefreshBackoffInterval = 30 + @data_hash = {} @parsed_token_uri = nil @http_client = nil @@ -140,7 +142,7 @@ def get_access_token token_response = http_access_token.request(token_request) # Handle the case where the response is not 200 parsed_json = JSON.parse(token_response.body) - @token_expiry_time = Time.now + 29 * 60 # set the expiry time to be ~thirty minutes from current time + @token_expiry_time = Time.now + @@tokenRefreshBackoffInterval * 60 # set the expiry time to be ~thirty minutes from current time @cached_access_token = parsed_json["access_token"] @log.info "Successfully got access token" end @@ -152,7 +154,7 @@ def get_access_token sleep(retries) retry else - @get_access_token_backoff_expiry = Time.now + 29 * 60 + @get_access_token_backoff_expiry = Time.now + @@tokenRefreshBackoffInterval * 60 @log.info "@get_access_token_backoff_expiry set to #{@get_access_token_backoff_expiry}" ApplicationInsightsUtility.sendExceptionTelemetry(err, {"FeatureArea" => "MDM"}) end From 7184e996dd247665fb0e6e32a4c04a9739e1475e Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 24 Feb 2020 13:59:04 -0800 Subject: [PATCH 17/17] adding case insensitive check --- source/code/plugin/out_mdm.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/plugin/out_mdm.rb b/source/code/plugin/out_mdm.rb index 2f90b89ee..243251bca 100644 --- a/source/code/plugin/out_mdm.rb +++ b/source/code/plugin/out_mdm.rb @@ -85,7 +85,7 @@ def start sp_client_id = @data_hash["aadClientId"] sp_client_secret = @data_hash["aadClientSecret"] - if (!sp_client_id.nil? && !sp_client_id.empty? && sp_client_id != "msi") + if (!sp_client_id.nil? && !sp_client_id.empty? && sp_client_id.downcase != "msi") @useMsi = false aad_token_url = @@aad_token_url_template % {tenant_id: @data_hash["tenantId"]} @parsed_token_uri = URI.parse(aad_token_url)