From 02fa2fbf56fb824205e0729a8f5be20e44578827 Mon Sep 17 00:00:00 2001 From: rashmichandrashekar Date: Fri, 3 Jan 2020 15:35:33 -0800 Subject: [PATCH 1/8] CAdvisor secure port changes (#320) * cadvsior secure port changes * update to use secure/insecure port for cadvisor * telemetry changes --- installer/conf/telegraf.conf | 2 +- .../code/plugin/CAdvisorMetricsAPIClient.rb | 53 +++++++++++++++---- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/installer/conf/telegraf.conf b/installer/conf/telegraf.conf index cd22a56b4..f9dc3fb6a 100644 --- a/installer/conf/telegraf.conf +++ b/installer/conf/telegraf.conf @@ -531,7 +531,7 @@ [[inputs.prometheus]] name_prefix="container.azm.ms/" ## An array of urls to scrape metrics from. - urls = ["http://$NODE_IP:10255/metrics"] + urls = ["$CADVISOR_METRICS_URL"] fieldpass = ["kubelet_docker_operations", "kubelet_docker_operations_errors"] metric_version = 2 diff --git a/source/code/plugin/CAdvisorMetricsAPIClient.rb b/source/code/plugin/CAdvisorMetricsAPIClient.rb index be61b8b8f..06377db01 100644 --- a/source/code/plugin/CAdvisorMetricsAPIClient.rb +++ b/source/code/plugin/CAdvisorMetricsAPIClient.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true class CAdvisorMetricsAPIClient - require 'yajl/json_gem' + require "yajl/json_gem" require "logger" require "net/http" require "net/https" @@ -29,6 +29,8 @@ class CAdvisorMetricsAPIClient @dsPromFieldDropCount = ENV["TELEMETRY_DS_PROM_FIELDDROP_LENGTH"] @dsPromUrlCount = ENV["TELEMETRY_DS_PROM_URLS_LENGTH"] + @cAdvisorMetricsSecurePort = ENV["IS_SECURE_CADVISOR_PORT"] + @LogPath = "/var/opt/microsoft/docker-cimprov/log/kubernetes_perf_log.txt" @Log = Logger.new(@LogPath, 2, 10 * 1048576) #keep last 2 files, max log file size = 10M # @@rxBytesLast = nil @@ -63,13 +65,33 @@ def getSummaryStatsFromCAdvisor(winNode) response = nil @Log.info "Getting CAdvisor Uri" begin - cAdvisorUri = getCAdvisorUri(winNode) + cAdvisorSecurePort = true + # Check to see if omsagent needs to use 10255(insecure) port or 10250(secure) port + if @cAdvisorMetricsSecurePort && @cAdvisorMetricsSecurePort.value == false + cAdvisorSecurePort = false + else + + cAdvisorUri = getCAdvisorUri(winNode, cAdvisorSecurePort) + bearerToken = File.read("/var/run/secrets/kubernetes.io/serviceaccount/token") + if !cAdvisorUri.nil? uri = URI.parse(cAdvisorUri) - Net::HTTP.start(uri.host, uri.port, :use_ssl => false, :open_timeout => 20, :read_timeout => 40 ) do |http| - cAdvisorApiRequest = Net::HTTP::Get.new(uri.request_uri) - response = http.request(cAdvisorApiRequest) - @Log.info "Got response code #{response.code} from #{uri.request_uri}" + if cAdvisorSecurePort + Net::HTTP.start(uri.host, uri.port, + :use_ssl => true, :open_timeout => 20, :read_timeout => 40, + :ca_file => "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", + :verify_mode => OpenSSL::SSL::VERIFY_NONE) do |http| + cAdvisorApiRequest = Net::HTTP::Get.new(uri.request_uri) + cAdvisorApiRequest["Authorization"] = "Bearer #{bearerToken}" + response = http.request(cAdvisorApiRequest) + @Log.info "Got response code #{response.code} from #{uri.request_uri}" + end + else + Net::HTTP.start(uri.host, uri.port, :use_ssl => false, :open_timeout => 20, :read_timeout => 40) do |http| + cAdvisorApiRequest = Net::HTTP::Get.new(uri.request_uri) + response = http.request(cAdvisorApiRequest) + @Log.info "Got response code #{response.code} from #{uri.request_uri}" + end end end rescue => error @@ -81,9 +103,14 @@ def getSummaryStatsFromCAdvisor(winNode) return response end - def getCAdvisorUri(winNode) + def getCAdvisorUri(winNode, cAdvisorSecurePort) begin - defaultHost = "http://localhost:10255" + if cAdvisorSecurePort + defaultHost = "https://localhost:10250" + else + defaultHost = "http://localhost:10255" + end + relativeUri = "/stats/summary" if !winNode.nil? nodeIP = winNode["InternalIP"] @@ -92,7 +119,11 @@ def getCAdvisorUri(winNode) end if !nodeIP.nil? @Log.info("Using #{nodeIP + relativeUri} for CAdvisor Uri") - return "http://#{nodeIP}:10255" + relativeUri + if cAdvisorSecurePort + return "https://#{nodeIP}:10250" + relativeUri + else + return "http://#{nodeIP}:10255" + relativeUri + end else @Log.warn ("NODE_IP environment variable not set. Using default as : #{defaultHost + relativeUri} ") if !winNode.nil? @@ -104,7 +135,7 @@ def getCAdvisorUri(winNode) end end - def getMetrics(winNode: nil, metricTime: Time.now.utc.iso8601 ) + def getMetrics(winNode: nil, metricTime: Time.now.utc.iso8601) metricDataItems = [] begin cAdvisorStats = getSummaryStatsFromCAdvisor(winNode) @@ -211,6 +242,7 @@ def getContainerCpuMetricItems(metricJSON, hostName, cpuMetricNameToCollect, met telemetryProps["PodName"] = podName telemetryProps["ContainerName"] = containerName telemetryProps["Computer"] = hostName + telemetryProps["CAdvisorIsSecure"] = @cAdvisorMetricsSecurePort #telemetry about log collections settings if (File.file?(@configMapMountPath)) telemetryProps["clustercustomsettings"] = true @@ -412,6 +444,7 @@ def getContainerMemoryMetricItems(metricJSON, hostName, memoryMetricNameToCollec telemetryProps["PodName"] = podName telemetryProps["ContainerName"] = containerName telemetryProps["Computer"] = hostName + telemetryProps["CAdvisorIsSecure"] = @cAdvisorMetricsSecurePort ApplicationInsightsUtility.sendMetricTelemetry(metricNametoReturn, metricValue, telemetryProps) end end From 671d534a8fa771193e410bb45561191bbc495083 Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 3 Jan 2020 17:16:18 -0800 Subject: [PATCH 2/8] fix bug --- source/code/plugin/CAdvisorMetricsAPIClient.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/plugin/CAdvisorMetricsAPIClient.rb b/source/code/plugin/CAdvisorMetricsAPIClient.rb index 06377db01..75bcee287 100644 --- a/source/code/plugin/CAdvisorMetricsAPIClient.rb +++ b/source/code/plugin/CAdvisorMetricsAPIClient.rb @@ -69,7 +69,7 @@ def getSummaryStatsFromCAdvisor(winNode) # Check to see if omsagent needs to use 10255(insecure) port or 10250(secure) port if @cAdvisorMetricsSecurePort && @cAdvisorMetricsSecurePort.value == false cAdvisorSecurePort = false - else + end cAdvisorUri = getCAdvisorUri(winNode, cAdvisorSecurePort) bearerToken = File.read("/var/run/secrets/kubernetes.io/serviceaccount/token") From 3f94d544ec97697b28aca8ef890b8371daf92859 Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 3 Jan 2020 17:31:55 -0800 Subject: [PATCH 3/8] bug fix --- source/code/plugin/CAdvisorMetricsAPIClient.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/plugin/CAdvisorMetricsAPIClient.rb b/source/code/plugin/CAdvisorMetricsAPIClient.rb index 75bcee287..1df7ec7fd 100644 --- a/source/code/plugin/CAdvisorMetricsAPIClient.rb +++ b/source/code/plugin/CAdvisorMetricsAPIClient.rb @@ -67,7 +67,7 @@ def getSummaryStatsFromCAdvisor(winNode) begin cAdvisorSecurePort = true # Check to see if omsagent needs to use 10255(insecure) port or 10250(secure) port - if @cAdvisorMetricsSecurePort && @cAdvisorMetricsSecurePort.value == false + if !@cAdvisorMetricsSecurePort.nil? && !!@cAdvisorMetricsSecurePort == false cAdvisorSecurePort = false end From bd82a50fa0dca9abe5c95b424bfbd40a19c8323b Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 3 Jan 2020 17:59:56 -0800 Subject: [PATCH 4/8] changes --- source/code/plugin/CAdvisorMetricsAPIClient.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/code/plugin/CAdvisorMetricsAPIClient.rb b/source/code/plugin/CAdvisorMetricsAPIClient.rb index 1df7ec7fd..3edca8ddb 100644 --- a/source/code/plugin/CAdvisorMetricsAPIClient.rb +++ b/source/code/plugin/CAdvisorMetricsAPIClient.rb @@ -67,7 +67,7 @@ def getSummaryStatsFromCAdvisor(winNode) begin cAdvisorSecurePort = true # Check to see if omsagent needs to use 10255(insecure) port or 10250(secure) port - if !@cAdvisorMetricsSecurePort.nil? && !!@cAdvisorMetricsSecurePort == false + if !@cAdvisorMetricsSecurePort.nil? && @cAdvisorMetricsSecurePort == "false" cAdvisorSecurePort = false end @@ -76,7 +76,7 @@ def getSummaryStatsFromCAdvisor(winNode) if !cAdvisorUri.nil? uri = URI.parse(cAdvisorUri) - if cAdvisorSecurePort + if !!cAdvisorSecurePort == true Net::HTTP.start(uri.host, uri.port, :use_ssl => true, :open_timeout => 20, :read_timeout => 40, :ca_file => "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt", @@ -105,7 +105,7 @@ def getSummaryStatsFromCAdvisor(winNode) def getCAdvisorUri(winNode, cAdvisorSecurePort) begin - if cAdvisorSecurePort + if !!cAdvisorSecurePort == true defaultHost = "https://localhost:10250" else defaultHost = "http://localhost:10255" @@ -119,7 +119,7 @@ def getCAdvisorUri(winNode, cAdvisorSecurePort) end if !nodeIP.nil? @Log.info("Using #{nodeIP + relativeUri} for CAdvisor Uri") - if cAdvisorSecurePort + if !!cAdvisorSecurePort == true return "https://#{nodeIP}:10250" + relativeUri else return "http://#{nodeIP}:10255" + relativeUri From ed1a9cd1a7090e6af638071d1d907fd00ebfb356 Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 6 Jan 2020 12:04:55 -0800 Subject: [PATCH 5/8] Adding cadvisor uri log --- source/code/plugin/CAdvisorMetricsAPIClient.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/code/plugin/CAdvisorMetricsAPIClient.rb b/source/code/plugin/CAdvisorMetricsAPIClient.rb index 3edca8ddb..327f611be 100644 --- a/source/code/plugin/CAdvisorMetricsAPIClient.rb +++ b/source/code/plugin/CAdvisorMetricsAPIClient.rb @@ -73,7 +73,8 @@ def getSummaryStatsFromCAdvisor(winNode) cAdvisorUri = getCAdvisorUri(winNode, cAdvisorSecurePort) bearerToken = File.read("/var/run/secrets/kubernetes.io/serviceaccount/token") - + @Log.info "cAdvisorUri: #{cAdvisorUri}" + if !cAdvisorUri.nil? uri = URI.parse(cAdvisorUri) if !!cAdvisorSecurePort == true From 1b4a211cb725e806d032e458f840ddbf4505c2e5 Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 6 Jan 2020 15:53:46 -0800 Subject: [PATCH 6/8] switching defaults --- source/code/plugin/CAdvisorMetricsAPIClient.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/code/plugin/CAdvisorMetricsAPIClient.rb b/source/code/plugin/CAdvisorMetricsAPIClient.rb index 327f611be..1bd9adf34 100644 --- a/source/code/plugin/CAdvisorMetricsAPIClient.rb +++ b/source/code/plugin/CAdvisorMetricsAPIClient.rb @@ -65,16 +65,16 @@ def getSummaryStatsFromCAdvisor(winNode) response = nil @Log.info "Getting CAdvisor Uri" begin - cAdvisorSecurePort = true + cAdvisorSecurePort = false # Check to see if omsagent needs to use 10255(insecure) port or 10250(secure) port - if !@cAdvisorMetricsSecurePort.nil? && @cAdvisorMetricsSecurePort == "false" - cAdvisorSecurePort = false + if !@cAdvisorMetricsSecurePort.nil? && @cAdvisorMetricsSecurePort == "true" + cAdvisorSecurePort = true end cAdvisorUri = getCAdvisorUri(winNode, cAdvisorSecurePort) bearerToken = File.read("/var/run/secrets/kubernetes.io/serviceaccount/token") @Log.info "cAdvisorUri: #{cAdvisorUri}" - + if !cAdvisorUri.nil? uri = URI.parse(cAdvisorUri) if !!cAdvisorSecurePort == true From 432104c14c8be1d30c2dd7b3419e08e99e741fcd Mon Sep 17 00:00:00 2001 From: rashmichandrashekar Date: Mon, 6 Jan 2020 16:36:03 -0800 Subject: [PATCH 7/8] update readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 007f92d92..9779d0ecc 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,10 @@ additional questions or comments. Note : The agent version(s) below has dates (ciprod), which indicate the agent build dates (not release dates) +### 01/07/2020 - +##### Version microsoft/oms:ciprod01072020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod01072020 +- Switch between 10255(old) and 10250(new) ports for cadvisor for older and newer versions of kubernetes + ### 12/04/2019 - ##### Version microsoft/oms:ciprod12042019 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod12042019 - Fix scheduler for all input plugins From e05da6d2e8a21c7846a1ff60930a11fe0025a62e Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 6 Jan 2020 17:33:53 -0800 Subject: [PATCH 8/8] changes --- README.md | 3 +++ source/code/plugin/CAdvisorMetricsAPIClient.rb | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9779d0ecc..75b2d8665 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,10 @@ Note : The agent version(s) below has dates (ciprod), which indicate t ### 01/07/2020 - ##### Version microsoft/oms:ciprod01072020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod01072020 +## Code change log - Switch between 10255(old) and 10250(new) ports for cadvisor for older and newer versions of kubernetes +## Customer Impact +- Node cpu, node memory, container cpu and container memory metrics were obtained earlier by querying kubelet readonly port(http://$NODE_IP:10255). Agent now supports getting these metrics from kubelet port(https://$NODE_IP:10250) as well. During the agent startup, it checks for connectivity to kubelet port(https://$NODE_IP:10250), and if it fails the metrics source is defaulted to readonly port(http://$NODE_IP:10255). ### 12/04/2019 - ##### Version microsoft/oms:ciprod12042019 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod12042019 diff --git a/source/code/plugin/CAdvisorMetricsAPIClient.rb b/source/code/plugin/CAdvisorMetricsAPIClient.rb index 1bd9adf34..8b0105a6f 100644 --- a/source/code/plugin/CAdvisorMetricsAPIClient.rb +++ b/source/code/plugin/CAdvisorMetricsAPIClient.rb @@ -445,7 +445,6 @@ def getContainerMemoryMetricItems(metricJSON, hostName, memoryMetricNameToCollec telemetryProps["PodName"] = podName telemetryProps["ContainerName"] = containerName telemetryProps["Computer"] = hostName - telemetryProps["CAdvisorIsSecure"] = @cAdvisorMetricsSecurePort ApplicationInsightsUtility.sendMetricTelemetry(metricNametoReturn, metricValue, telemetryProps) end end