From ac3fe1663ced0ca7a5fbd5ec3e6a97a9dcb60921 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 8 Feb 2020 23:52:44 -0800 Subject: [PATCH 01/73] trim log tag for cri compatible log lines --- source/code/go/src/plugins/oms.go | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 8dfaf0e7e..424e8d223 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -37,6 +37,9 @@ const ResourceIdEnv = "AKS_RESOURCE_ID" //env variable which has ResourceName for NON-AKS const ResourceNameEnv = "ACS_RESOURCE_NAME" +//env variable which has container run time name +const ContainerRuntimeEnv = "CONTAINER_RUN_TIME" + // Origin prefix for telegraf Metrics (used as prefix for origin field & prefix for azure monitor specific tags and also for custom-metrics telemetry ) const TelegrafMetricOriginPrefix = "container.azm.ms" @@ -94,6 +97,8 @@ var ( skipKubeMonEventsFlush bool // enrich container logs (when true this will add the fields - timeofcommand, containername & containerimage) enrichContainerLogs bool + // container runtime engine configured on the kubelet (true indicates docker else K8s CRI compatiable runtime) + isDockerContainerRuntimeEngine bool ) var ( @@ -734,7 +739,14 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { stringMap := make(map[string]string) - stringMap["LogEntry"] = ToString(record["log"]) + logEntry := ToString(record["log"]) + if isDockerContainerRuntimeEngine == false { + //CRI compatiable runtimes uses the CRIO parser which has logtag followed space and then log line + // so trimming the log tag (P or F) followed space + logEntry = strings.TrimPrefix(logEntry, "P ") + logEntry = strings.TrimPrefix(logEntry, "F ") + } + stringMap["LogEntry"] = logEntry stringMap["LogEntrySource"] = logEntrySource stringMap["LogEntryTimeStamp"] = ToString(record["time"]) stringMap["SourceSystem"] = "Containers" @@ -958,6 +970,12 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { Log("ResourceID=%s", ResourceID) Log("ResourceName=%s", ResourceName) } + + containerRuntime = os.Getenv(ContainerRuntimeEnv) + isDockerContainerRuntimeEngine = false + if strings.EqualFold(containerRuntime, "docker") { + isDockerContainerRuntimeEngine = true + } // Initialize image,name map refresh ticker containerInventoryRefreshInterval, err := strconv.Atoi(pluginConfig["container_inventory_refresh_interval"]) From 2f7673d23da2adc6f77dafc036ab138d1b04e479 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 9 Feb 2020 10:17:11 -0800 Subject: [PATCH 02/73] fix variable declaration --- source/code/go/src/plugins/oms.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 424e8d223..d98281183 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -971,7 +971,7 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { Log("ResourceName=%s", ResourceName) } - containerRuntime = os.Getenv(ContainerRuntimeEnv) + containerRuntime := os.Getenv(ContainerRuntimeEnv) isDockerContainerRuntimeEngine = false if strings.EqualFold(containerRuntime, "docker") { isDockerContainerRuntimeEngine = true From 8756d775bd253ab6a2144c4e12018a20a779487f Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 10 Feb 2020 22:58:51 -0800 Subject: [PATCH 03/73] debug log messages --- source/code/go/src/plugins/oms.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index d98281183..a4628de2a 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -99,6 +99,8 @@ var ( enrichContainerLogs bool // container runtime engine configured on the kubelet (true indicates docker else K8s CRI compatiable runtime) isDockerContainerRuntimeEngine bool + // container runtime engine configured on the kubelet (true indicates docker else K8s CRI compatiable runtime) + containerRuntime string ) var ( @@ -743,8 +745,10 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { if isDockerContainerRuntimeEngine == false { //CRI compatiable runtimes uses the CRIO parser which has logtag followed space and then log line // so trimming the log tag (P or F) followed space + Log("Container Runtime engine %s isDockerContainerRuntimeEngine %t", containerRuntime, isDockerContainerRuntimeEngine) logEntry = strings.TrimPrefix(logEntry, "P ") logEntry = strings.TrimPrefix(logEntry, "F ") + Log("After trimming LogEntry %s", logEntry) } stringMap["LogEntry"] = logEntry stringMap["LogEntrySource"] = logEntrySource @@ -793,7 +797,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { if dataItem.LogEntryTimeStamp != "" { loggedTime, e := time.Parse(time.RFC3339, dataItem.LogEntryTimeStamp) if e != nil { - message := fmt.Sprintf("Error while converting LogEntryTimeStamp for telemetry purposes: %s", e.Error()) + message := fmt.Sprintf("containerId: %s Error while converting LogEntryTimeStamp for telemetry purposes: %s", dataItem.ID, e.Error()) Log(message) SendException(message) } else { @@ -971,7 +975,8 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { Log("ResourceName=%s", ResourceName) } - containerRuntime := os.Getenv(ContainerRuntimeEnv) + containerRuntime = os.Getenv(ContainerRuntimeEnv) + Log("Container Runtime engine %s", containerRuntime) isDockerContainerRuntimeEngine = false if strings.EqualFold(containerRuntime, "docker") { isDockerContainerRuntimeEngine = true From 3c1a34720f0ef00fb89d7ea52e28ef825c48c634 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 11 Feb 2020 01:01:09 -0800 Subject: [PATCH 04/73] trim spaces --- source/code/go/src/plugins/oms.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index a4628de2a..035d752c5 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -746,9 +746,13 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { //CRI compatiable runtimes uses the CRIO parser which has logtag followed space and then log line // so trimming the log tag (P or F) followed space Log("Container Runtime engine %s isDockerContainerRuntimeEngine %t", containerRuntime, isDockerContainerRuntimeEngine) - logEntry = strings.TrimPrefix(logEntry, "P ") - logEntry = strings.TrimPrefix(logEntry, "F ") - Log("After trimming LogEntry %s", logEntry) + logEntry = strings.TrimSpace(logEntry) + if strings.HasPrefix(logEntry, "P ") { + logEntry = strings.TrimPrefix(logEntry, "P ") + } else { + logEntry = strings.TrimPrefix(logEntry, "F ") + } + Log("After trimming LogEntry:%s", logEntry) } stringMap["LogEntry"] = logEntry stringMap["LogEntrySource"] = logEntrySource From eb89239abfa2d89cd39d135fa4bcfaea8ecf2933 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 11 Feb 2020 01:20:47 -0800 Subject: [PATCH 05/73] remove debug logs --- source/code/go/src/plugins/oms.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 035d752c5..586c0a057 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -742,21 +742,21 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { stringMap := make(map[string]string) logEntry := ToString(record["log"]) + logEntryTimeStamp := ToString(record["time"]) if isDockerContainerRuntimeEngine == false { //CRI compatiable runtimes uses the CRIO parser which has logtag followed space and then log line - // so trimming the log tag (P or F) followed space - Log("Container Runtime engine %s isDockerContainerRuntimeEngine %t", containerRuntime, isDockerContainerRuntimeEngine) + // so trimming the log tag (P or F) logEntry = strings.TrimSpace(logEntry) if strings.HasPrefix(logEntry, "P ") { logEntry = strings.TrimPrefix(logEntry, "P ") } else { logEntry = strings.TrimPrefix(logEntry, "F ") - } - Log("After trimming LogEntry:%s", logEntry) + } + logEntryTimeStamp = strings.Trimspace(logEntryTimeStamp) } stringMap["LogEntry"] = logEntry stringMap["LogEntrySource"] = logEntrySource - stringMap["LogEntryTimeStamp"] = ToString(record["time"]) + stringMap["LogEntryTimeStamp"] = logEntryTimeStamp stringMap["SourceSystem"] = "Containers" stringMap["Id"] = containerID From 54a99f2629b2f12e5ceac03b36feae6818502a23 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 11 Feb 2020 01:24:57 -0800 Subject: [PATCH 06/73] fix build error --- source/code/go/src/plugins/oms.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 586c0a057..4e7bd6917 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -752,7 +752,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { } else { logEntry = strings.TrimPrefix(logEntry, "F ") } - logEntryTimeStamp = strings.Trimspace(logEntryTimeStamp) + logEntryTimeStamp = strings.TrimSpace(logEntryTimeStamp) } stringMap["LogEntry"] = logEntry stringMap["LogEntrySource"] = logEntrySource From c2f6609e56055623d8a00a85f1ec24fdf204fedb Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 12 Feb 2020 14:32:00 -0800 Subject: [PATCH 07/73] add pods api in cadvisor --- source/code/plugin/CAdvisorMetricsAPIClient.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/code/plugin/CAdvisorMetricsAPIClient.rb b/source/code/plugin/CAdvisorMetricsAPIClient.rb index 54e7e5fd9..979fd3bbc 100644 --- a/source/code/plugin/CAdvisorMetricsAPIClient.rb +++ b/source/code/plugin/CAdvisorMetricsAPIClient.rb @@ -72,6 +72,11 @@ def getNodeCapacityFromCAdvisor(winNode: nil) return getResponse(winNode, relativeUri) end + def getPodsFromCAdvisor(winNode: nil) + relativeUri = "/pods" + return getResponse(winNode, relativeUri) + end + def getBaseCAdvisorUri(winNode) cAdvisorSecurePort = isCAdvisorOnSecurePort() From 1eb2e0a78f68b306d215934b33c06d39261bdcfc Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 12 Feb 2020 16:02:20 -0800 Subject: [PATCH 08/73] wip --- source/code/plugin/kubelet_utils.rb | 31 +++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 6d97e30a9..0055d3136 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -19,5 +19,36 @@ def get_node_capacity return [cpu_capacity, memory_capacity] end end + + def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + containerInventoryRecords = Array.new + response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) + if !response.nil? && !response.body.nil? + podList = JSON.parse(response.body) + if !podList.nil? && !podList.empty? + podList["items"].each do |item| + nodeName = item["spec"]["nodeName"] + containerInstance = {} + status = item.status + if !status.nil? && !status.empty? + containerStatuses = status.containerStatuses + if !containerStatuses.nil? && !containerStatuses.empty? + containerStatuses.each do |item| + containerInstance["ElementName"] = containerStatuses["name"] + containerInstance["InstanceID"] = container["containerID"] + containerInstance["Computer"] = nodeName + containerInstance["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + end + end + end + containerSpec = item.spec + containerInventoryRecords.push containerInstance + end + end + end + + return containerInventoryRecords + end + end end \ No newline at end of file From d647a416747c1722b860cc523134ba02d5a306ef Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 13 Feb 2020 01:27:18 -0800 Subject: [PATCH 09/73] wip --- source/code/plugin/kubelet_utils.rb | 86 ++++++++++++++++++++--------- 1 file changed, 61 insertions(+), 25 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 0055d3136..24bf02550 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -21,34 +21,70 @@ def get_node_capacity end def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) - containerInventoryRecords = Array.new - response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) - if !response.nil? && !response.body.nil? - podList = JSON.parse(response.body) - if !podList.nil? && !podList.empty? - podList["items"].each do |item| - nodeName = item["spec"]["nodeName"] - containerInstance = {} - status = item.status - if !status.nil? && !status.empty? - containerStatuses = status.containerStatuses - if !containerStatuses.nil? && !containerStatuses.empty? - containerStatuses.each do |item| - containerInstance["ElementName"] = containerStatuses["name"] - containerInstance["InstanceID"] = container["containerID"] - containerInstance["Computer"] = nodeName - containerInstance["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated - end - end - end - containerSpec = item.spec - containerInventoryRecords.push containerInstance - end + containerInventoryRecords = Array.new + begin + response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) + if !response.nil? && !response.body.nil? + podList = JSON.parse(response.body) + if !podList.nil? && !podList.empty? + podList["items"].each do |item| + containersInfoMap = getContainersInfoMap(spec, clusterCollectEnvironmentVar) + containerInstance = {} + if !item["status"].nil? && !item["status"].empty? + if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? + item["status"]["containerStatuses"].each do |containerStatus| + containerInstance["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + containerName = containerStatus["name"] + containerInstance["ContainerID"] = containerStatus["containerID"].split('//')[0] + containerInfoMap = containersInfoMap[containerName] + containerInstance["ElementName"] = containerInfoMap["ElementName"] + containerInstance["Computer"] = containerInfoMap["Computer"] + containerInstance["ContainerHostname"] = containerInfoMap["ContainerHostname"] + containerInstance["CreatedTime"] = containerInfoMap["CreatedTime"] + containerInstance["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] + containerInstance["Ports"] = containerInfoMap["Ports"] + containerInstance["Command"] = containerInfoMap["Command"] + containerInventoryRecords.push containerInstance + end + end + end + end + end end - end - + rescue => error + @Log.warn("kubelet_utils::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") + end return containerInventoryRecords end + def getContainersInfoMap(item, clusterCollectEnvironmentVar) + containersInfoMap = {} + begin + createdTime = item["metadata"]["creationTimestamp"] + nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" + item["spec"]["containers"].each do |container| + containerInfoMap = {} + containerName = container["name"] + containerInfoMap["ElementName"] = containerName + containerInfoMap["Computer"] = nodeName + containerInfoMap["ContainerHostname"] = nodeName + containerInfoMap["CreatedTime"] = createdTime + envValue = container["env"] + envValueString = (envValue.nil?) ? "" : envValue.to_s + containerInfoMap["EnvironmentVar"] = envValueString + portsValue = container["ports"] + portsValueString = (portsValue.nil?) ? "" : portsValue.to_s + containerInfoMap["Ports"] = portsValueString + argsValue = container["args"] + argsValueString = (argsValue.nil?) ? "" : argsValue.to_s + containerInfoMap["Command"] = argsValueString + + containersInfoMap[containerName] = containerInfoMap + end + rescue => error + @Log.warn("kubelet_utils::getContainersInfoMap : Get Container Info Maps failed: #{error}") + end + return containersInfoMap + end end end \ No newline at end of file From 93568434f0a722d330f5656c76a60114e8a9bae6 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 13 Feb 2020 12:41:29 -0800 Subject: [PATCH 10/73] wip --- source/code/plugin/kubelet_utils.rb | 94 +++++++++++++++++++---------- 1 file changed, 63 insertions(+), 31 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 24bf02550..21b511c82 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -26,25 +26,32 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) if !response.nil? && !response.body.nil? podList = JSON.parse(response.body) - if !podList.nil? && !podList.empty? + if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? podList["items"].each do |item| - containersInfoMap = getContainersInfoMap(spec, clusterCollectEnvironmentVar) - containerInstance = {} + containersInfoMap = getContainersInfoMap(item, clusterCollectEnvironmentVar) + containerInventoryRecord = {} if !item["status"].nil? && !item["status"].empty? if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? item["status"]["containerStatuses"].each do |containerStatus| - containerInstance["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated containerName = containerStatus["name"] - containerInstance["ContainerID"] = containerStatus["containerID"].split('//')[0] + containerInventoryRecord["ContainerID"] = containerStatus["containerID"].split('//')[1] + repoImageTagArray = containerStatus["image"].split("/") + containerInventoryRecord["Repository"] = repoImageTagArray[0] + imageTagArray = repoImageTagArray[1].split(":") + containerInventoryRecord["Image"] = imageTagArray[0] + containerInventoryRecord["ImageTag"] = imageTagArray[1] + # Setting the image id to the id in the remote repository + containerInventoryRecord["ImageId"] = containerStatus["imageID"].split("@")[1] containerInfoMap = containersInfoMap[containerName] - containerInstance["ElementName"] = containerInfoMap["ElementName"] - containerInstance["Computer"] = containerInfoMap["Computer"] - containerInstance["ContainerHostname"] = containerInfoMap["ContainerHostname"] - containerInstance["CreatedTime"] = containerInfoMap["CreatedTime"] - containerInstance["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] - containerInstance["Ports"] = containerInfoMap["Ports"] - containerInstance["Command"] = containerInfoMap["Command"] - containerInventoryRecords.push containerInstance + containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] + containerInventoryRecord["Computer"] = containerInfoMap["Computer"] + containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] + containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] + containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] + containerInventoryRecord["Ports"] = containerInfoMap["Ports"] + containerInventoryRecord["Command"] = containerInfoMap["Command"] + containerInventoryRecords.push containerInventoryRecord end end end @@ -60,26 +67,51 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) def getContainersInfoMap(item, clusterCollectEnvironmentVar) containersInfoMap = {} begin - createdTime = item["metadata"]["creationTimestamp"] nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" - item["spec"]["containers"].each do |container| - containerInfoMap = {} - containerName = container["name"] - containerInfoMap["ElementName"] = containerName - containerInfoMap["Computer"] = nodeName - containerInfoMap["ContainerHostname"] = nodeName - containerInfoMap["CreatedTime"] = createdTime - envValue = container["env"] - envValueString = (envValue.nil?) ? "" : envValue.to_s - containerInfoMap["EnvironmentVar"] = envValueString - portsValue = container["ports"] - portsValueString = (portsValue.nil?) ? "" : portsValue.to_s - containerInfoMap["Ports"] = portsValueString - argsValue = container["args"] - argsValueString = (argsValue.nil?) ? "" : argsValue.to_s - containerInfoMap["Command"] = argsValueString + createdTime = item["metadata"]["creationTimestamp"] + if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? + if !item["spec"]["containers"].nil? && !item["spec"]["containers"].empty? + item["spec"]["containers"].each do |container| + containerInfoMap = {} + containerName = container["name"] + containerInfoMap["ElementName"] = containerName + containerInfoMap["Computer"] = nodeName + containerInfoMap["ContainerHostname"] = nodeName + containerInfoMap["CreatedTime"] = createdTime - containersInfoMap[containerName] = containerInfoMap + if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 + containerInfoMap["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] + else + envValue = container["env"] + envValueString = (envValue.nil?) ? "" : envValue.to_s + # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE + # Check to see if the environment variable collection is disabled for this container. + if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) + envValueString = ["AZMON_COLLECT_ENV=FALSE"] + $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") + end + # Restricting the ENV string value to 200kb since the size of this string can go very high + if envValueString.length > 200000 + envValueStringTruncated = envValueString.slice(0..200000) + lastIndex = envValueStringTruncated.rindex("\", ") + if !lastIndex.nil? + envValueStringTruncated = envValueStringTruncated.slice(0..lastIndex) + "]" + end + containerInfoMap["EnvironmentVar"] = envValueStringTruncated + else + containerInfoMap["EnvironmentVar"] = envValueString + end + end + portsValue = container["ports"] + portsValueString = (portsValue.nil?) ? "" : portsValue.to_s + containerInfoMap["Ports"] = portsValueString + argsValue = container["args"] + argsValueString = (argsValue.nil?) ? "" : argsValue.to_s + containerInfoMap["Command"] = argsValueString + + containersInfoMap[containerName] = containerInfoMap + end + end end rescue => error @Log.warn("kubelet_utils::getContainersInfoMap : Get Container Info Maps failed: #{error}") From 6cf3e50acc14485fc13cc3a30de313aefc9b2e9a Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 13 Feb 2020 13:26:05 -0800 Subject: [PATCH 11/73] wip --- source/code/plugin/in_containerinventory.rb | 89 ++++++++++++++------- source/code/plugin/kubelet_utils.rb | 15 ++++ 2 files changed, 76 insertions(+), 28 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index d107047b4..f2d7767a9 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -19,6 +19,7 @@ def initialize require_relative "ContainerInventoryState" require_relative "ApplicationInsightsUtility" require_relative "omslog" + require_relative "kubelet_utils" end config_param :run_interval, :time, :default => 60 @@ -195,37 +196,69 @@ def enumerate emitTime = currentTime.to_f batchTime = currentTime.utc.iso8601 containerInventory = Array.new - $log.info("in_container_inventory::enumerate : Begin processing @ #{Time.now.utc.iso8601}") - hostname = DockerApiClient.getDockerHostName + hostname = "" + $log.info("in_container_inventory::enumerate : Begin processing @ #{Time.now.utc.iso8601}") begin - containerIds = DockerApiClient.listContainers - if !containerIds.nil? && !containerIds.empty? - eventStream = MultiEventStream.new - nameMap = DockerApiClient.getImageIdMap - clusterCollectEnvironmentVar = ENV["AZMON_CLUSTER_COLLECT_ENV_VAR"] - if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 - $log.warn("Environment Variable collection disabled for cluster") - end - containerIds.each do |containerId| - inspectedContainer = {} - inspectedContainer = inspectContainer(containerId, nameMap, clusterCollectEnvironmentVar) - inspectedContainer["Computer"] = hostname - inspectedContainer["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated - containerInventory.push inspectedContainer - ContainerInventoryState.writeContainerState(inspectedContainer) - end - # Update the state for deleted containers - deletedContainers = ContainerInventoryState.getDeletedContainers(containerIds) - if !deletedContainers.nil? && !deletedContainers.empty? - deletedContainers.each do |deletedContainer| - container = ContainerInventoryState.readContainerState(deletedContainer) - if !container.nil? - container.each { |k, v| container[k] = v } - container["State"] = "Deleted" - containerInventory.push container + containerRuntimeEnv = ENV["CONTAINER_RUN_TIME"] + $log.info("in_container_inventory::enumerate : container runtime #{containerRuntimeEnv}") + + clusterCollectEnvironmentVar = ENV["AZMON_CLUSTER_COLLECT_ENV_VAR"] + if !containerRuntimeEnv.nil? && !containerRuntimeEnv.empty? && containerRuntimeEnv.casecmp("docker") + $log.info("in_container_inventory::enumerate : using docker sock since container runtime is docker") + hostname = DockerApiClient.getDockerHostName + containerIds = DockerApiClient.listContainers + if !containerIds.nil? && !containerIds.empty? + eventStream = MultiEventStream.new + nameMap = DockerApiClient.getImageIdMap + if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 + $log.warn("Environment Variable collection disabled for cluster") + end + containerIds.each do |containerId| + inspectedContainer = {} + inspectedContainer = inspectContainer(containerId, nameMap, clusterCollectEnvironmentVar) + inspectedContainer["Computer"] = hostname + inspectedContainer["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + containerInventory.push inspectedContainer + ContainerInventoryState.writeContainerState(inspectedContainer) + end + # Update the state for deleted containers + deletedContainers = ContainerInventoryState.getDeletedContainers(containerIds) + if !deletedContainers.nil? && !deletedContainers.empty? + deletedContainers.each do |deletedContainer| + container = ContainerInventoryState.readContainerState(deletedContainer) + if !container.nil? + container.each { |k, v| container[k] = v } + container["State"] = "Deleted" + containerInventory.push container + end end end - end + else + $log.info("in_container_inventory::enumerate : using kubelet apis since CRI compatiable runtime") + containerInventoryRecords = kubelet_utils.getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + containerIds = Array.new + containerInventoryRecords.each do |containerRecord| + ContainerInventoryState.writeContainerState(containerRecord) + if !hostName.empty? && !containerRecord["Computer"].empty? + hostName = containerRecord["Computer"] + end + containerIds.push containerRecord["ContainerID"] + containerInventory.push containerRecord + end + # Update the state for deleted containers + deletedContainers = ContainerInventoryState.getDeletedContainers(containerIds) + if !deletedContainers.nil? && !deletedContainers.empty? + deletedContainers.each do |deletedContainer| + container = ContainerInventoryState.readContainerState(deletedContainer) + if !container.nil? + container.each { |k, v| container[k] = v } + container["State"] = "Deleted" + containerInventory.push container + end + end + end + + end containerInventory.each do |record| wrapper = { diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 21b511c82..e6e3d09fd 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -43,6 +43,21 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerInventoryRecord["ImageTag"] = imageTagArray[1] # Setting the image id to the id in the remote repository containerInventoryRecord["ImageId"] = containerStatus["imageID"].split("@")[1] + containerInventoryRecord["ExitCode"] = 0 + if !containerStatus["state"].nil? && !containerStatus["state"].empty? + containerState = containerStatus["state"] + if containerState.key?("running") + containerInventoryRecord["State"] = "Running" + containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] + elsif containerState.key?("terminated") + containerInventoryRecord["State"] = "Terminated" + containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] + containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] + containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] + elsif containerState.key?("waiting") + containerInventoryRecord["State"] = "Waiting" + end + end containerInfoMap = containersInfoMap[containerName] containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] containerInventoryRecord["Computer"] = containerInfoMap["Computer"] From 358513fe7c08df0d96cab192524f03912e41187a Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 13 Feb 2020 14:27:19 -0800 Subject: [PATCH 12/73] fix bug --- source/code/plugin/in_containerinventory.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index f2d7767a9..ae485a908 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -203,7 +203,7 @@ def enumerate $log.info("in_container_inventory::enumerate : container runtime #{containerRuntimeEnv}") clusterCollectEnvironmentVar = ENV["AZMON_CLUSTER_COLLECT_ENV_VAR"] - if !containerRuntimeEnv.nil? && !containerRuntimeEnv.empty? && containerRuntimeEnv.casecmp("docker") + if !containerRuntimeEnv.nil? && !containerRuntimeEnv.empty? && containerRuntimeEnv.casecmp("docker") == 0 $log.info("in_container_inventory::enumerate : using docker sock since container runtime is docker") hostname = DockerApiClient.getDockerHostName containerIds = DockerApiClient.listContainers From 8f41eb2596ace4619fe4a2f12b5362f7175d36ad Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 13 Feb 2020 15:05:43 -0800 Subject: [PATCH 13/73] fix bug --- source/code/plugin/in_containerinventory.rb | 111 ++++++++++---------- 1 file changed, 55 insertions(+), 56 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index ae485a908..7f3713827 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -200,80 +200,79 @@ def enumerate $log.info("in_container_inventory::enumerate : Begin processing @ #{Time.now.utc.iso8601}") begin containerRuntimeEnv = ENV["CONTAINER_RUN_TIME"] - $log.info("in_container_inventory::enumerate : container runtime #{containerRuntimeEnv}") - + $log.info("in_container_inventory::enumerate : container runtime #{containerRuntimeEnv}") clusterCollectEnvironmentVar = ENV["AZMON_CLUSTER_COLLECT_ENV_VAR"] if !containerRuntimeEnv.nil? && !containerRuntimeEnv.empty? && containerRuntimeEnv.casecmp("docker") == 0 - $log.info("in_container_inventory::enumerate : using docker sock since container runtime is docker") - hostname = DockerApiClient.getDockerHostName - containerIds = DockerApiClient.listContainers - if !containerIds.nil? && !containerIds.empty? - eventStream = MultiEventStream.new - nameMap = DockerApiClient.getImageIdMap - if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 - $log.warn("Environment Variable collection disabled for cluster") - end - containerIds.each do |containerId| - inspectedContainer = {} - inspectedContainer = inspectContainer(containerId, nameMap, clusterCollectEnvironmentVar) - inspectedContainer["Computer"] = hostname - inspectedContainer["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated - containerInventory.push inspectedContainer - ContainerInventoryState.writeContainerState(inspectedContainer) - end - # Update the state for deleted containers - deletedContainers = ContainerInventoryState.getDeletedContainers(containerIds) - if !deletedContainers.nil? && !deletedContainers.empty? - deletedContainers.each do |deletedContainer| - container = ContainerInventoryState.readContainerState(deletedContainer) - if !container.nil? - container.each { |k, v| container[k] = v } - container["State"] = "Deleted" - containerInventory.push container + $log.info("in_container_inventory::enumerate : using docker sock since container runtime is docker") + hostname = DockerApiClient.getDockerHostName + containerIds = DockerApiClient.listContainers + if !containerIds.nil? && !containerIds.empty? + eventStream = MultiEventStream.new + nameMap = DockerApiClient.getImageIdMap + if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 + $log.warn("Environment Variable collection disabled for cluster") + end + containerIds.each do |containerId| + inspectedContainer = {} + inspectedContainer = inspectContainer(containerId, nameMap, clusterCollectEnvironmentVar) + inspectedContainer["Computer"] = hostname + inspectedContainer["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + containerInventory.push inspectedContainer + ContainerInventoryState.writeContainerState(inspectedContainer) + end + # Update the state for deleted containers + deletedContainers = ContainerInventoryState.getDeletedContainers(containerIds) + if !deletedContainers.nil? && !deletedContainers.empty? + deletedContainers.each do |deletedContainer| + container = ContainerInventoryState.readContainerState(deletedContainer) + if !container.nil? + container.each { |k, v| container[k] = v } + container["State"] = "Deleted" + containerInventory.push container + end end end - end - else + end + else $log.info("in_container_inventory::enumerate : using kubelet apis since CRI compatiable runtime") containerInventoryRecords = kubelet_utils.getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerIds = Array.new containerInventoryRecords.each do |containerRecord| ContainerInventoryState.writeContainerState(containerRecord) - if !hostName.empty? && !containerRecord["Computer"].empty? - hostName = containerRecord["Computer"] + if hostName.empty? && !containerRecord["Computer"].empty? + hostName = containerRecord["Computer"] end containerIds.push containerRecord["ContainerID"] containerInventory.push containerRecord end # Update the state for deleted containers deletedContainers = ContainerInventoryState.getDeletedContainers(containerIds) - if !deletedContainers.nil? && !deletedContainers.empty? - deletedContainers.each do |deletedContainer| - container = ContainerInventoryState.readContainerState(deletedContainer) - if !container.nil? - container.each { |k, v| container[k] = v } - container["State"] = "Deleted" - containerInventory.push container + if !deletedContainers.nil? && !deletedContainers.empty? + deletedContainers.each do |deletedContainer| + container = ContainerInventoryState.readContainerState(deletedContainer) + if !container.nil? + container.each { |k, v| container[k] = v } + container["State"] = "Deleted" + containerInventory.push container + end end end - end - - end + end - containerInventory.each do |record| - wrapper = { - "DataType" => "CONTAINER_INVENTORY_BLOB", - "IPName" => "ContainerInsights", - "DataItems" => [record.each { |k, v| record[k] = v }], - } - eventStream.add(emitTime, wrapper) if wrapper - end - router.emit_stream(@tag, eventStream) if eventStream - @@istestvar = ENV["ISTEST"] - if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0 && eventStream.count > 0) - $log.info("containerInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") - end - $log.info("in_container_inventory::enumerate : Processing complete - emitted stream @ #{Time.now.utc.iso8601}") + containerInventory.each do |record| + wrapper = { + "DataType" => "CONTAINER_INVENTORY_BLOB", + "IPName" => "ContainerInsights", + "DataItems" => [record.each { |k, v| record[k] = v }], + } + eventStream.add(emitTime, wrapper) if wrapper + end + router.emit_stream(@tag, eventStream) if eventStream + @@istestvar = ENV["ISTEST"] + if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0 && eventStream.count > 0) + $log.info("containerInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") + end + $log.info("in_container_inventory::enumerate : Processing complete - emitted stream @ #{Time.now.utc.iso8601}") end timeDifference = (DateTime.now.to_time.to_i - @@telemetryTimeTracker).abs timeDifferenceInMinutes = timeDifference / 60 From 889a37db4d05062ee5c430aef5488f89965fe5ec Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 13 Feb 2020 15:44:25 -0800 Subject: [PATCH 14/73] fix bug with end --- source/code/plugin/in_containerinventory.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 7f3713827..cf01605c7 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -272,8 +272,7 @@ def enumerate if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0 && eventStream.count > 0) $log.info("containerInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") end - $log.info("in_container_inventory::enumerate : Processing complete - emitted stream @ #{Time.now.utc.iso8601}") - end + $log.info("in_container_inventory::enumerate : Processing complete - emitted stream @ #{Time.now.utc.iso8601}") timeDifference = (DateTime.now.to_time.to_i - @@telemetryTimeTracker).abs timeDifferenceInMinutes = timeDifference / 60 if (timeDifferenceInMinutes >= 5) From fbeb65c19c0d22da6faaaff63360119ccb4e4479 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 13 Feb 2020 16:11:53 -0800 Subject: [PATCH 15/73] fix bug in the reference --- source/code/plugin/in_containerinventory.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index cf01605c7..bd859a349 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -235,7 +235,7 @@ def enumerate end else $log.info("in_container_inventory::enumerate : using kubelet apis since CRI compatiable runtime") - containerInventoryRecords = kubelet_utils.getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + containerInventoryRecords = KubeletUtils.getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerIds = Array.new containerInventoryRecords.each do |containerRecord| ContainerInventoryState.writeContainerState(containerRecord) From 6ca988106cb9a71bb2fd9d38423b288e00be41e9 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 13 Feb 2020 19:28:00 -0800 Subject: [PATCH 16/73] fix bug --- source/code/plugin/in_containerinventory.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index bd859a349..c3343e0a6 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -196,7 +196,8 @@ def enumerate emitTime = currentTime.to_f batchTime = currentTime.utc.iso8601 containerInventory = Array.new - hostname = "" + eventStream = MultiEventStream.new + hostName = "" $log.info("in_container_inventory::enumerate : Begin processing @ #{Time.now.utc.iso8601}") begin containerRuntimeEnv = ENV["CONTAINER_RUN_TIME"] @@ -204,10 +205,9 @@ def enumerate clusterCollectEnvironmentVar = ENV["AZMON_CLUSTER_COLLECT_ENV_VAR"] if !containerRuntimeEnv.nil? && !containerRuntimeEnv.empty? && containerRuntimeEnv.casecmp("docker") == 0 $log.info("in_container_inventory::enumerate : using docker sock since container runtime is docker") - hostname = DockerApiClient.getDockerHostName + hostName = DockerApiClient.getDockerHostName containerIds = DockerApiClient.listContainers - if !containerIds.nil? && !containerIds.empty? - eventStream = MultiEventStream.new + if !containerIds.nil? && !containerIds.empty? nameMap = DockerApiClient.getImageIdMap if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 $log.warn("Environment Variable collection disabled for cluster") @@ -215,7 +215,7 @@ def enumerate containerIds.each do |containerId| inspectedContainer = {} inspectedContainer = inspectContainer(containerId, nameMap, clusterCollectEnvironmentVar) - inspectedContainer["Computer"] = hostname + inspectedContainer["Computer"] = hostName inspectedContainer["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated containerInventory.push inspectedContainer ContainerInventoryState.writeContainerState(inspectedContainer) @@ -278,7 +278,7 @@ def enumerate if (timeDifferenceInMinutes >= 5) @@telemetryTimeTracker = DateTime.now.to_time.to_i telemetryProperties = {} - telemetryProperties["Computer"] = hostname + telemetryProperties["Computer"] = hostName telemetryProperties["ContainerCount"] = containerInventory.length ApplicationInsightsUtility.sendTelemetry(@@PluginName, telemetryProperties) end From 3798163ac6197289077b3d1b44f5c43dd386aa0b Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 13 Feb 2020 21:30:35 -0800 Subject: [PATCH 17/73] fix telemetry --- .../code/plugin/ApplicationInsightsUtility.rb | 35 +++++++++++-------- source/code/plugin/in_containerinventory.rb | 6 ++-- source/code/plugin/in_kube_nodes.rb | 12 ++++--- 3 files changed, 32 insertions(+), 21 deletions(-) diff --git a/source/code/plugin/ApplicationInsightsUtility.rb b/source/code/plugin/ApplicationInsightsUtility.rb index f7bd806a0..1bb4db282 100644 --- a/source/code/plugin/ApplicationInsightsUtility.rb +++ b/source/code/plugin/ApplicationInsightsUtility.rb @@ -20,6 +20,7 @@ class ApplicationInsightsUtility @@EnvApplicationInsightsKey = "APPLICATIONINSIGHTS_AUTH" @@EnvApplicationInsightsEndpoint = "APPLICATIONINSIGHTS_ENDPOINT" @@EnvControllerType = "CONTROLLER_TYPE" + @@EnvContainerRuntime = "CONTAINER_RUN_TIME" @@CustomProperties = {} @@Tc = nil @@ -64,7 +65,7 @@ def initializeUtility() @@CustomProperties["ControllerType"] = ENV[@@EnvControllerType] encodedAppInsightsKey = ENV[@@EnvApplicationInsightsKey] appInsightsEndpoint = ENV[@@EnvApplicationInsightsEndpoint] - @@CustomProperties["WorkspaceCloud"] = getWorkspaceCloud + @@CustomProperties["WorkspaceCloud"] = getWorkspaceCloud #Check if telemetry is turned off telemetryOffSwitch = ENV["DISABLE_TELEMETRY"] @@ -108,15 +109,21 @@ def initializeUtility() rescue => errorStr $log.warn("Exception in AppInsightsUtility: initilizeUtility - error: #{errorStr}") end - end + end - def getDockerInfo() - dockerInfo = DockerApiClient.dockerInfo - if (!dockerInfo.nil? && !dockerInfo.empty?) - @@CustomProperties["DockerVersion"] = dockerInfo["Version"] - #@@CustomProperties["DockerApiVersion"] = dockerInfo["ApiVersion"] + def getContainerRuntimeInfo() + containerRuntime = ENV[@@EnvContainerRuntime] + if !containerRuntime.nil? && !containerRuntime.empty? + @@CustomProperties["ContainerRunTime"] = containerRuntime + if containerRuntime.casecmp("docker") == 0 + dockerInfo = DockerApiClient.dockerInfo + if (!dockerInfo.nil? && !dockerInfo.empty?) + @@CustomProperties["DockerVersion"] = dockerInfo["Version"] + #@@CustomProperties["DockerApiVersion"] = dockerInfo["ApiVersion"] + end + end end - end + end def sendHeartBeatEvent(pluginName) begin @@ -168,8 +175,8 @@ def sendExceptionTelemetry(errorStr, properties = nil) begin if @@CustomProperties.empty? || @@CustomProperties.nil? initializeUtility() - elsif @@CustomProperties["DockerVersion"].nil? - getDockerInfo() + elsif @@CustomProperties["ContainerRunTime"].nil? + getContainerRuntimeInfo() end telemetryProps = {} # add common dimensions @@ -192,8 +199,8 @@ def sendTelemetry(pluginName, properties) begin if @@CustomProperties.empty? || @@CustomProperties.nil? initializeUtility() - elsif @@CustomProperties["DockerVersion"].nil? - getDockerInfo() + elsif @@CustomProperties["ContainerRunTime"].nil? + getContainerRuntimeInfo() end @@CustomProperties["Computer"] = properties["Computer"] sendHeartBeatEvent(pluginName) @@ -212,8 +219,8 @@ def sendMetricTelemetry(metricName, metricValue, properties) end if @@CustomProperties.empty? || @@CustomProperties.nil? initializeUtility() - elsif @@CustomProperties["DockerVersion"].nil? - getDockerInfo() + elsif @@CustomProperties["ContainerRunTime"].nil? + getContainerRuntimeInfo() end telemetryProps = {} # add common dimensions diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index c3343e0a6..8e47ce7e9 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -201,10 +201,10 @@ def enumerate $log.info("in_container_inventory::enumerate : Begin processing @ #{Time.now.utc.iso8601}") begin containerRuntimeEnv = ENV["CONTAINER_RUN_TIME"] - $log.info("in_container_inventory::enumerate : container runtime #{containerRuntimeEnv}") + $log.info("in_container_inventory::enumerate : container runtime : #{containerRuntimeEnv}") clusterCollectEnvironmentVar = ENV["AZMON_CLUSTER_COLLECT_ENV_VAR"] if !containerRuntimeEnv.nil? && !containerRuntimeEnv.empty? && containerRuntimeEnv.casecmp("docker") == 0 - $log.info("in_container_inventory::enumerate : using docker sock since container runtime is docker") + $log.info("in_container_inventory::enumerate : using docker apis since container runtime is docker") hostName = DockerApiClient.getDockerHostName containerIds = DockerApiClient.listContainers if !containerIds.nil? && !containerIds.empty? @@ -234,7 +234,7 @@ def enumerate end end else - $log.info("in_container_inventory::enumerate : using kubelet apis since CRI compatiable runtime") + $log.info("in_container_inventory::enumerate : using cadvisor apis since container runtime is non docker") containerInventoryRecords = KubeletUtils.getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerIds = Array.new containerInventoryRecords.each do |containerRecord| diff --git a/source/code/plugin/in_kube_nodes.rb b/source/code/plugin/in_kube_nodes.rb index 706c3ad13..6a099d626 100644 --- a/source/code/plugin/in_kube_nodes.rb +++ b/source/code/plugin/in_kube_nodes.rb @@ -160,9 +160,10 @@ def parse_and_emit_records(nodeInventory, batchTime = Time.utc.iso8601) record["KubeletVersion"] = nodeInfo["kubeletVersion"] record["KubeProxyVersion"] = nodeInfo["kubeProxyVersion"] containerNodeInventoryRecord["OperatingSystem"] = nodeInfo["osImage"] - dockerVersion = nodeInfo["containerRuntimeVersion"] - dockerVersion.slice! "docker://" - containerNodeInventoryRecord["DockerVersion"] = dockerVersion + containerRuntimeVersion = nodeInfo["containerRuntimeVersion"] + if containerRuntimeVersion.downcase.start_with?("docker://") + containerNodeInventoryRecord["DockerVersion"] = containerRuntimeVersion.split("//")[1] + end # ContainerNodeInventory data for docker version and operating system. containerNodeInventoryWrapper = { "DataType" => "CONTAINER_NODE_INVENTORY_BLOB", @@ -185,7 +186,10 @@ def parse_and_emit_records(nodeInventory, batchTime = Time.utc.iso8601) properties["Computer"] = record["Computer"] properties["KubeletVersion"] = record["KubeletVersion"] properties["OperatingSystem"] = nodeInfo["operatingSystem"] - properties["DockerVersion"] = dockerVersion + properties["ContainerRuntimeVersion"] = containerRuntimeVersion + if containerRuntimeVersion.downcase.start_with?("docker://") + properties["DockerVersion"] = containerRuntimeVersion.split("//")[1] + end properties["KubernetesProviderID"] = record["KubernetesProviderID"] properties["KernelVersion"] = nodeInfo["kernelVersion"] properties["OSImage"] = nodeInfo["osImage"] From df952764921d9ca763ddebc3fd395767bbbbb500 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 14 Feb 2020 14:37:44 -0800 Subject: [PATCH 18/73] fix image and imageid bug --- source/code/plugin/in_containerinventory.rb | 26 ++--- source/code/plugin/kubelet_utils.rb | 102 ++++++++++++-------- 2 files changed, 76 insertions(+), 52 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 8e47ce7e9..68106fa88 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -198,17 +198,17 @@ def enumerate containerInventory = Array.new eventStream = MultiEventStream.new hostName = "" - $log.info("in_container_inventory::enumerate : Begin processing @ #{Time.now.utc.iso8601}") + $log.info("in_container_inventory::enumerate : Begin processing @ #{Time.now.utc.iso8601}") begin containerRuntimeEnv = ENV["CONTAINER_RUN_TIME"] - $log.info("in_container_inventory::enumerate : container runtime : #{containerRuntimeEnv}") + $log.info("in_container_inventory::enumerate : container runtime : #{containerRuntimeEnv}") clusterCollectEnvironmentVar = ENV["AZMON_CLUSTER_COLLECT_ENV_VAR"] if !containerRuntimeEnv.nil? && !containerRuntimeEnv.empty? && containerRuntimeEnv.casecmp("docker") == 0 - $log.info("in_container_inventory::enumerate : using docker apis since container runtime is docker") + $log.info("in_container_inventory::enumerate : using docker apis since container runtime is docker") hostName = DockerApiClient.getDockerHostName containerIds = DockerApiClient.listContainers - if !containerIds.nil? && !containerIds.empty? - nameMap = DockerApiClient.getImageIdMap + if !containerIds.nil? && !containerIds.empty? + nameMap = DockerApiClient.getImageIdMap if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 $log.warn("Environment Variable collection disabled for cluster") end @@ -232,19 +232,19 @@ def enumerate end end end - end + end else - $log.info("in_container_inventory::enumerate : using cadvisor apis since container runtime is non docker") - containerInventoryRecords = KubeletUtils.getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + $log.info("in_container_inventory::enumerate : using cadvisor apis since container runtime is non docker") + containerInventoryRecords = KubeletUtils.getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerIds = Array.new - containerInventoryRecords.each do |containerRecord| + containerInventoryRecords.each do |containerRecord| ContainerInventoryState.writeContainerState(containerRecord) if hostName.empty? && !containerRecord["Computer"].empty? hostName = containerRecord["Computer"] - end + end containerIds.push containerRecord["ContainerID"] containerInventory.push containerRecord - end + end # Update the state for deleted containers deletedContainers = ContainerInventoryState.getDeletedContainers(containerIds) if !deletedContainers.nil? && !deletedContainers.empty? @@ -257,7 +257,7 @@ def enumerate end end end - end + end containerInventory.each do |record| wrapper = { @@ -272,7 +272,7 @@ def enumerate if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0 && eventStream.count > 0) $log.info("containerInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") end - $log.info("in_container_inventory::enumerate : Processing complete - emitted stream @ #{Time.now.utc.iso8601}") + $log.info("in_container_inventory::enumerate : Processing complete - emitted stream @ #{Time.now.utc.iso8601}") timeDifference = (DateTime.now.to_time.to_i - @@telemetryTimeTracker).abs timeDifferenceInMinutes = timeDifference / 60 if (timeDifferenceInMinutes >= 5) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index e6e3d09fd..0f803840b 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -7,7 +7,7 @@ class KubeletUtils class << self def get_node_capacity - + cpu_capacity = 1.0 memory_capacity = 1.0 @@ -20,8 +20,8 @@ def get_node_capacity end end - def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) - containerInventoryRecords = Array.new + def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + containerInventoryRecords = Array.new begin response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) if !response.nil? && !response.body.nil? @@ -29,22 +29,46 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? podList["items"].each do |item| containersInfoMap = getContainersInfoMap(item, clusterCollectEnvironmentVar) - containerInventoryRecord = {} - if !item["status"].nil? && !item["status"].empty? + containerInventoryRecord = {} + if !item["status"].nil? && !item["status"].empty? if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? - item["status"]["containerStatuses"].each do |containerStatus| - containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated - containerName = containerStatus["name"] + item["status"]["containerStatuses"].each do |containerStatus| + containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + containerName = containerStatus["name"] containerInventoryRecord["ContainerID"] = containerStatus["containerID"].split('//')[1] - repoImageTagArray = containerStatus["image"].split("/") - containerInventoryRecord["Repository"] = repoImageTagArray[0] - imageTagArray = repoImageTagArray[1].split(":") - containerInventoryRecord["Image"] = imageTagArray[0] - containerInventoryRecord["ImageTag"] = imageTagArray[1] - # Setting the image id to the id in the remote repository - containerInventoryRecord["ImageId"] = containerStatus["imageID"].split("@")[1] + # imagedId is of the format - repo@sha256:imageid + imageIdValue = containerStatus["imageID"] + if !imageIdValue.nil? && !imageIdValue.empty? + atLocation = imageIdValue.index("@") + if !atLocation.nil? + containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] + end + end + # image is of the format - repository/image:imagetag + imageValue = containerStatus["image"] + if !imageValue.nil? && !imageValue.empty? + # Find delimiters in the string of format repository/image:imagetag + slashLocation = imageValue.index("/") + colonLocation = imageValue.index(":") + if !colonLocation.nil? + if slashLocation.nil? + # image:imagetag + containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] + else + # repository/image:imagetag + containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] + containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] + end + containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] + end + elsif !imageIdValue.nil? && !imageIdValue.empty? + # Getting repo information from imageIdValue when no tag in ImageId + if !atLocation.nil? + containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] + end + end containerInventoryRecord["ExitCode"] = 0 - if !containerStatus["state"].nil? && !containerStatus["state"].empty? + if !containerStatus["state"].nil? && !containerStatus["state"].empty? containerState = containerStatus["state"] if containerState.key?("running") containerInventoryRecord["State"] = "Running" @@ -56,9 +80,9 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] elsif containerState.key?("waiting") containerInventoryRecord["State"] = "Waiting" - end + end end - containerInfoMap = containersInfoMap[containerName] + containerInfoMap = containersInfoMap[containerName] containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] containerInventoryRecord["Computer"] = containerInfoMap["Computer"] containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] @@ -68,29 +92,29 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerInventoryRecord["Command"] = containerInfoMap["Command"] containerInventoryRecords.push containerInventoryRecord end - end - end - end + end + end + end end end rescue => error @Log.warn("kubelet_utils::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") - end + end return containerInventoryRecords end def getContainersInfoMap(item, clusterCollectEnvironmentVar) containersInfoMap = {} begin - nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" - createdTime = item["metadata"]["creationTimestamp"] - if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? + nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" + createdTime = item["metadata"]["creationTimestamp"] + if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? if !item["spec"]["containers"].nil? && !item["spec"]["containers"].empty? item["spec"]["containers"].each do |container| containerInfoMap = {} - containerName = container["name"] + containerName = container["name"] containerInfoMap["ElementName"] = containerName - containerInfoMap["Computer"] = nodeName + containerInfoMap["Computer"] = nodeName containerInfoMap["ContainerHostname"] = nodeName containerInfoMap["CreatedTime"] = createdTime @@ -102,8 +126,8 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE # Check to see if the environment variable collection is disabled for this container. if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) - envValueString = ["AZMON_COLLECT_ENV=FALSE"] - $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") + envValueString = ["AZMON_COLLECT_ENV=FALSE"] + $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") end # Restricting the ENV string value to 200kb since the size of this string can go very high if envValueString.length > 200000 @@ -116,22 +140,22 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) else containerInfoMap["EnvironmentVar"] = envValueString end - end + end portsValue = container["ports"] - portsValueString = (portsValue.nil?) ? "" : portsValue.to_s - containerInfoMap["Ports"] = portsValueString + portsValueString = (portsValue.nil?) ? "" : portsValue.to_s + containerInfoMap["Ports"] = portsValueString argsValue = container["args"] - argsValueString = (argsValue.nil?) ? "" : argsValue.to_s + argsValueString = (argsValue.nil?) ? "" : argsValue.to_s containerInfoMap["Command"] = argsValueString - containersInfoMap[containerName] = containerInfoMap - end + containersInfoMap[containerName] = containerInfoMap + end end - end - rescue => error - @Log.warn("kubelet_utils::getContainersInfoMap : Get Container Info Maps failed: #{error}") + end + rescue => error + @Log.warn("kubelet_utils::getContainersInfoMap : Get Container Info Maps failed: #{error}") end return containersInfoMap - end + end end end \ No newline at end of file From fff43515079eda2df7bda153f5502dfb407a4f65 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 15 Feb 2020 12:47:38 -0800 Subject: [PATCH 19/73] fix containerid bug --- source/code/plugin/in_containerinventory.rb | 2 +- source/code/plugin/kubelet_utils.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 68106fa88..ac57e1811 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -242,7 +242,7 @@ def enumerate if hostName.empty? && !containerRecord["Computer"].empty? hostName = containerRecord["Computer"] end - containerIds.push containerRecord["ContainerID"] + containerIds.push containerRecord["InstanceID"] containerInventory.push containerRecord end # Update the state for deleted containers diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 0f803840b..345a6a90c 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -35,7 +35,7 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) item["status"]["containerStatuses"].each do |containerStatus| containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated containerName = containerStatus["name"] - containerInventoryRecord["ContainerID"] = containerStatus["containerID"].split('//')[1] + containerInventoryRecord["InstanceID"] = containerStatus["containerID"].split('//')[1] # imagedId is of the format - repo@sha256:imageid imageIdValue = containerStatus["imageID"] if !imageIdValue.nil? && !imageIdValue.empty? From 40a915d1bb53a85e1cc30ac268d3d13eb1b0dad6 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 15 Feb 2020 21:02:20 -0800 Subject: [PATCH 20/73] fix envvars --- source/code/plugin/kubelet_utils.rb | 47 ++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 345a6a90c..11781dc0f 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -120,27 +120,44 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 containerInfoMap["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] - else - envValue = container["env"] - envValueString = (envValue.nil?) ? "" : envValue.to_s + else + envVarsJSON = container["env"] + envValueString = "" + envVars = [] + if !envVarsJSON.nil? && !envVarsJSON.empty? + envVarsJSON.each do |envVar| + key = envVar["name"] + if !envVar["value"].nil? + value = envVar["value"] + elsif !envVar["ValueFrom"].nil? + value = env["valueFrom"].to_s + else + value = "" + end + envVars.push("#{key}=#{value}") + end + envValueString = envVars.to_s + end # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE # Check to see if the environment variable collection is disabled for this container. if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) envValueString = ["AZMON_COLLECT_ENV=FALSE"] $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") + else + # Restricting the ENV string value to 200kb since the size of this string can go very high + if envValueString.length > 200000 + envValueStringTruncated = envValueString.slice(0..200000) + lastIndex = envValueStringTruncated.rindex("\", ") + if !lastIndex.nil? + envValueStringTruncated = envValueStringTruncated.slice(0..lastIndex) + "]" + end + containerInfoMap["EnvironmentVar"] = envValueStringTruncated + else + containerInfoMap["EnvironmentVar"] = envValueString + end + end end - # Restricting the ENV string value to 200kb since the size of this string can go very high - if envValueString.length > 200000 - envValueStringTruncated = envValueString.slice(0..200000) - lastIndex = envValueStringTruncated.rindex("\", ") - if !lastIndex.nil? - envValueStringTruncated = envValueStringTruncated.slice(0..lastIndex) + "]" - end - containerInfoMap["EnvironmentVar"] = envValueStringTruncated - else - containerInfoMap["EnvironmentVar"] = envValueString - end - end + end portsValue = container["ports"] portsValueString = (portsValue.nil?) ? "" : portsValue.to_s containerInfoMap["Ports"] = portsValueString From cdaa9e8cb888e958ff6afac9184f693dcfdf538b Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 15 Feb 2020 21:32:10 -0800 Subject: [PATCH 21/73] fix syntax error --- source/code/plugin/kubelet_utils.rb | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 11781dc0f..3438b8b75 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -137,7 +137,7 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) envVars.push("#{key}=#{value}") end envValueString = envVars.to_s - end + end # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE # Check to see if the environment variable collection is disabled for this container. if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) @@ -150,12 +150,11 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) lastIndex = envValueStringTruncated.rindex("\", ") if !lastIndex.nil? envValueStringTruncated = envValueStringTruncated.slice(0..lastIndex) + "]" - end - containerInfoMap["EnvironmentVar"] = envValueStringTruncated - else - containerInfoMap["EnvironmentVar"] = envValueString - end - end + end + containerInfoMap["EnvironmentVar"] = envValueStringTruncated + else + containerInfoMap["EnvironmentVar"] = envValueString + end end end portsValue = container["ports"] From f17dd9175ffdd5a7d221dc229dc13c2811aa6e59 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 15 Feb 2020 22:07:17 -0800 Subject: [PATCH 22/73] fix syntax error --- source/code/plugin/kubelet_utils.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 3438b8b75..934ca4d6e 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -156,7 +156,8 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) containerInfoMap["EnvironmentVar"] = envValueString end end - end + end + portsValue = container["ports"] portsValueString = (portsValue.nil?) ? "" : portsValue.to_s containerInfoMap["Ports"] = portsValueString From 50404aa90e81017d368564af96a7ea96b9d69202 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Feb 2020 12:28:52 -0800 Subject: [PATCH 23/73] fix env and command --- source/code/plugin/kubelet_utils.rb | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 934ca4d6e..4a4f100c8 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -129,8 +129,18 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) key = envVar["name"] if !envVar["value"].nil? value = envVar["value"] - elsif !envVar["ValueFrom"].nil? - value = env["valueFrom"].to_s + elsif !envVar["valueFrom"].nil? + valueFrom = envVar["valueFrom"] + value = "" + if valueFrom.key?("fieldRef") && !valueFrom["fieldRef"]["fieldPath"].nil? && !valueFrom["fieldRef"]["fieldPath"].empty? + fieldPath = valueFrom["fieldRef"]["fieldPath"] + fields = fieldPath.split('.') + if fields.length() == 2 + value = item[fields[0]][fields[1]] + end + else + value = envVar["valueFrom"].to_s + end else value = "" end @@ -157,13 +167,13 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) end end end - + portsValue = container["ports"] portsValueString = (portsValue.nil?) ? "" : portsValue.to_s containerInfoMap["Ports"] = portsValueString - argsValue = container["args"] - argsValueString = (argsValue.nil?) ? "" : argsValue.to_s - containerInfoMap["Command"] = argsValueString + cmdValue = container["command"] + cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s + containerInfoMap["Command"] = cmdValueString containersInfoMap[containerName] = containerInfoMap end From 8bf65ccecd8a94ee2b701ab28c4a2452cd4d33e7 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 17 Feb 2020 17:15:16 -0800 Subject: [PATCH 24/73] handle labels and annotation fieldref as envs --- , | 0 source/code/plugin/kubelet_utils.rb | 19 +++++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 , diff --git a/, b/, new file mode 100644 index 000000000..e69de29bb diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 4a4f100c8..4dd7e2d56 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -127,22 +127,29 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) if !envVarsJSON.nil? && !envVarsJSON.empty? envVarsJSON.each do |envVar| key = envVar["name"] + value = "" if !envVar["value"].nil? value = envVar["value"] elsif !envVar["valueFrom"].nil? - valueFrom = envVar["valueFrom"] - value = "" + valueFrom = envVar["valueFrom"] if valueFrom.key?("fieldRef") && !valueFrom["fieldRef"]["fieldPath"].nil? && !valueFrom["fieldRef"]["fieldPath"].empty? fieldPath = valueFrom["fieldRef"]["fieldPath"] fields = fieldPath.split('.') if fields.length() == 2 - value = item[fields[0]][fields[1]] + if !fields[1].nil? && !fields[1].empty? & fields[1].end_with?(']') + indexFields = fields[1].split('[') + hashMapValue = item[fields[0]][indexFields[0]] + if !hashMapValue.nil? && !hashMapValue.empty? + subField = indexFields[1].delete_suffix("]").delete("\\'") + value = hashMapValue[subField] + end + else + value = item[fields[0]][fields[1]] + end end else value = envVar["valueFrom"].to_s - end - else - value = "" + end end envVars.push("#{key}=#{value}") end From 809bf77e0572e080136226fa37a866ad1a8f47ea Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 19 Feb 2020 14:57:08 -0800 Subject: [PATCH 25/73] wip --- source/code/plugin/in_containerinventory.rb | 2 + source/code/plugin/kubelet_utils.rb | 85 ++++++++++++++++----- 2 files changed, 68 insertions(+), 19 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index ac57e1811..c986c36d6 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -36,6 +36,8 @@ def start @mutex = Mutex.new @thread = Thread.new(&method(:run_periodic)) @@telemetryTimeTracker = DateTime.now.to_time.to_i + # cache the container and cgroup parent process + @containerCGroupCache = Hash.new end end diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 4dd7e2d56..101ecbf02 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -120,7 +120,7 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 containerInfoMap["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] - else + else envVarsJSON = container["env"] envValueString = "" envVars = [] @@ -130,50 +130,50 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) value = "" if !envVar["value"].nil? value = envVar["value"] - elsif !envVar["valueFrom"].nil? - valueFrom = envVar["valueFrom"] + elsif !envVar["valueFrom"].nil? + valueFrom = envVar["valueFrom"] if valueFrom.key?("fieldRef") && !valueFrom["fieldRef"]["fieldPath"].nil? && !valueFrom["fieldRef"]["fieldPath"].empty? fieldPath = valueFrom["fieldRef"]["fieldPath"] fields = fieldPath.split('.') if fields.length() == 2 if !fields[1].nil? && !fields[1].empty? & fields[1].end_with?(']') - indexFields = fields[1].split('[') - hashMapValue = item[fields[0]][indexFields[0]] - if !hashMapValue.nil? && !hashMapValue.empty? - subField = indexFields[1].delete_suffix("]").delete("\\'") - value = hashMapValue[subField] + indexFields = fields[1].split('[') + hashMapValue = item[fields[0]][indexFields[0]] + if !hashMapValue.nil? && !hashMapValue.empty? + subField = indexFields[1].delete_suffix("]").delete("\\'") + value = hashMapValue[subField] end - else - value = item[fields[0]][fields[1]] + else + value = item[fields[0]][fields[1]] end - end - else + end + else value = envVar["valueFrom"].to_s - end - end + end + end envVars.push("#{key}=#{value}") end envValueString = envVars.to_s - end + end # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE # Check to see if the environment variable collection is disabled for this container. if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) envValueString = ["AZMON_COLLECT_ENV=FALSE"] $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") - else + else # Restricting the ENV string value to 200kb since the size of this string can go very high if envValueString.length > 200000 envValueStringTruncated = envValueString.slice(0..200000) lastIndex = envValueStringTruncated.rindex("\", ") if !lastIndex.nil? envValueStringTruncated = envValueStringTruncated.slice(0..lastIndex) + "]" - end + end containerInfoMap["EnvironmentVar"] = envValueStringTruncated else containerInfoMap["EnvironmentVar"] = envValueString - end + end end - end + end portsValue = container["ports"] portsValueString = (portsValue.nil?) ? "" : portsValue.to_s @@ -191,5 +191,52 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) end return containersInfoMap end + def getContainerEnvironmentVars(containerId, containerCgroupCache) + + begin + unless containerCgroupCache.has_key?(containerId) + Dir["/hostfs/proc/*/cgroup"].each do| filename| + if File.file?(filename) && File.foreach(filename).grep(/#{containerId}/).any? + # file full path is /hostfs/proc//cgroup + cGroupPid = filename.split("/")[3] + if containerCgroupCache.has_key?(containerId) + temCGroupPid = containerCgroupCache[containerId] + if temCGroupPid > cGroupPid + containerCgroupCache[containerId] = cGroupPid + end + else + containerCgroupCache[containerId] = cGroupPid + end + end + end + end + cGroupPid = containerCgroupCache[containerId] + environFilePath = "/hostfs/proc/#{cGroupPid}/environ" + envVars = File.read(environFilePath).split(" ") + envValueString = envVars.to_s + # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE + # Check to see if the environment variable collection is disabled for this container. + if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) + envValueString = ["AZMON_COLLECT_ENV=FALSE"] + $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") + else + # Restricting the ENV string value to 200kb since the size of this string can go very high + if envValueString.length > 200000 + envValueStringTruncated = envValueString.slice(0..200000) + lastIndex = envValueStringTruncated.rindex("\", ") + if !lastIndex.nil? + envValueStringTruncated = envValueStringTruncated.slice(0..lastIndex) + "]" + end + containerInfoMap["EnvironmentVar"] = envValueStringTruncated + else + containerInfoMap["EnvironmentVar"] = envValueString + end + end + + rescue => error + @Log.warn("kubelet_utils::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") + end + end + end end \ No newline at end of file From 99fa994015438fca7b624079bf47ac342662e631 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 19 Feb 2020 23:24:43 -0800 Subject: [PATCH 26/73] implement obtain envvars --- source/code/plugin/in_containerinventory.rb | 5 +- source/code/plugin/kubelet_utils.rb | 137 +++++++------------- 2 files changed, 47 insertions(+), 95 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index c986c36d6..717d1627c 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -237,9 +237,9 @@ def enumerate end else $log.info("in_container_inventory::enumerate : using cadvisor apis since container runtime is non docker") - containerInventoryRecords = KubeletUtils.getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + containerInventoryRecords = KubeletUtils.getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar, @containerCGroupCache) containerIds = Array.new - containerInventoryRecords.each do |containerRecord| + containerInventoryRecords.each do |containerRecord| ContainerInventoryState.writeContainerState(containerRecord) if hostName.empty? && !containerRecord["Computer"].empty? hostName = containerRecord["Computer"] @@ -255,6 +255,7 @@ def enumerate if !container.nil? container.each { |k, v| container[k] = v } container["State"] = "Deleted" + @containerCGroupCache.delete(container["InstanceID"]) containerInventory.push container end end diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 101ecbf02..54accb512 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -20,7 +20,7 @@ def get_node_capacity end end - def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar, @containerCGroupCache) containerInventoryRecords = Array.new begin response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) @@ -28,14 +28,15 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) podList = JSON.parse(response.body) if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? podList["items"].each do |item| - containersInfoMap = getContainersInfoMap(item, clusterCollectEnvironmentVar) + containersInfoMap = getContainersInfoMap(item) containerInventoryRecord = {} if !item["status"].nil? && !item["status"].empty? if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? item["status"]["containerStatuses"].each do |containerStatus| containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated containerName = containerStatus["name"] - containerInventoryRecord["InstanceID"] = containerStatus["containerID"].split('//')[1] + containerId = containerStatus["containerID"].split('//')[1] + containerInventoryRecord["InstanceID"] = containerId # imagedId is of the format - repo@sha256:imageid imageIdValue = containerStatus["imageID"] if !imageIdValue.nil? && !imageIdValue.empty? @@ -90,6 +91,11 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] containerInventoryRecord["Ports"] = containerInfoMap["Ports"] containerInventoryRecord["Command"] = containerInfoMap["Command"] + if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 + containerInventoryRecord["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] + else + containerInventoryRecord["EnvironmentVar"] = KubeletUtils.obtainContainerEnvironmentVars(containerId, @containerCGroupCache) + end containerInventoryRecords.push containerInventoryRecord end end @@ -103,7 +109,7 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) return containerInventoryRecords end - def getContainersInfoMap(item, clusterCollectEnvironmentVar) + def getContainersInfoMap(item) containersInfoMap = {} begin nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" @@ -116,65 +122,7 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) containerInfoMap["ElementName"] = containerName containerInfoMap["Computer"] = nodeName containerInfoMap["ContainerHostname"] = nodeName - containerInfoMap["CreatedTime"] = createdTime - - if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 - containerInfoMap["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] - else - envVarsJSON = container["env"] - envValueString = "" - envVars = [] - if !envVarsJSON.nil? && !envVarsJSON.empty? - envVarsJSON.each do |envVar| - key = envVar["name"] - value = "" - if !envVar["value"].nil? - value = envVar["value"] - elsif !envVar["valueFrom"].nil? - valueFrom = envVar["valueFrom"] - if valueFrom.key?("fieldRef") && !valueFrom["fieldRef"]["fieldPath"].nil? && !valueFrom["fieldRef"]["fieldPath"].empty? - fieldPath = valueFrom["fieldRef"]["fieldPath"] - fields = fieldPath.split('.') - if fields.length() == 2 - if !fields[1].nil? && !fields[1].empty? & fields[1].end_with?(']') - indexFields = fields[1].split('[') - hashMapValue = item[fields[0]][indexFields[0]] - if !hashMapValue.nil? && !hashMapValue.empty? - subField = indexFields[1].delete_suffix("]").delete("\\'") - value = hashMapValue[subField] - end - else - value = item[fields[0]][fields[1]] - end - end - else - value = envVar["valueFrom"].to_s - end - end - envVars.push("#{key}=#{value}") - end - envValueString = envVars.to_s - end - # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE - # Check to see if the environment variable collection is disabled for this container. - if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) - envValueString = ["AZMON_COLLECT_ENV=FALSE"] - $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") - else - # Restricting the ENV string value to 200kb since the size of this string can go very high - if envValueString.length > 200000 - envValueStringTruncated = envValueString.slice(0..200000) - lastIndex = envValueStringTruncated.rindex("\", ") - if !lastIndex.nil? - envValueStringTruncated = envValueStringTruncated.slice(0..lastIndex) + "]" - end - containerInfoMap["EnvironmentVar"] = envValueStringTruncated - else - containerInfoMap["EnvironmentVar"] = envValueString - end - end - end - + containerInfoMap["CreatedTime"] = createdTime portsValue = container["ports"] portsValueString = (portsValue.nil?) ? "" : portsValue.to_s containerInfoMap["Ports"] = portsValueString @@ -191,51 +139,54 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) end return containersInfoMap end - def getContainerEnvironmentVars(containerId, containerCgroupCache) - begin - unless containerCgroupCache.has_key?(containerId) + def obtainContainerEnvironmentVars(containerId, containerCGroupCache) + envValueString = "" + begin + unless containerCGroupCache.has_key?(containerId) Dir["/hostfs/proc/*/cgroup"].each do| filename| if File.file?(filename) && File.foreach(filename).grep(/#{containerId}/).any? # file full path is /hostfs/proc//cgroup cGroupPid = filename.split("/")[3] - if containerCgroupCache.has_key?(containerId) + if containerCGroupCache.has_key?(containerId) temCGroupPid = containerCgroupCache[containerId] if temCGroupPid > cGroupPid containerCgroupCache[containerId] = cGroupPid end else - containerCgroupCache[containerId] = cGroupPid + containerCGroupCache[containerId] = cGroupPid end end end end - cGroupPid = containerCgroupCache[containerId] - environFilePath = "/hostfs/proc/#{cGroupPid}/environ" - envVars = File.read(environFilePath).split(" ") - envValueString = envVars.to_s - # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE - # Check to see if the environment variable collection is disabled for this container. - if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) - envValueString = ["AZMON_COLLECT_ENV=FALSE"] - $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") - else - # Restricting the ENV string value to 200kb since the size of this string can go very high - if envValueString.length > 200000 - envValueStringTruncated = envValueString.slice(0..200000) - lastIndex = envValueStringTruncated.rindex("\", ") - if !lastIndex.nil? - envValueStringTruncated = envValueStringTruncated.slice(0..lastIndex) + "]" + cGroupPid = containerCGroupCache[containerId] + if !cGroupPid.nil? + environFilePath = "/hostfs/proc/#{cGroupPid}/environ" + if File.exist?(environFilePath) + # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE + # Check to see if the environment variable collection is disabled for this container. + if File.foreach(environFilePath).grep(/AZMON_COLLECT_ENV=FALSE/i).any? + envValueString = ["AZMON_COLLECT_ENV=FALSE"] + $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") + else + fileSize = File.size(environFilePath) + # Restricting the ENV string value to 200kb since the size of this string can go very high + envVars = File.read(environFilePath, 200000).split(" ") + envValueString = envVars.to_s + if fileSize > 200000 + lastIndex = envValueString.rindex("\", ") + if !lastIndex.nil? + envValueStringTruncated = envValueString.slice(0..lastIndex) + "]" + envValueString = envValueStringTruncated + end + end end - containerInfoMap["EnvironmentVar"] = envValueStringTruncated - else - containerInfoMap["EnvironmentVar"] = envValueString - end - end - - rescue => error - @Log.warn("kubelet_utils::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") - end + end + end + rescue => error + @Log.warn("kubelet_utils::obtainContainerEnvironmentVars : obtain Container Environment vars failed: #{error} for containerId: #{containerId}") + end + return envValueString end end From 6f70046fe9b66238950ad946051c1ab3f810c3be Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 20 Feb 2020 12:01:49 -0800 Subject: [PATCH 27/73] refactor the code --- source/code/plugin/in_containerinventory.rb | 181 +++++++++++++++++++- source/code/plugin/kubelet_utils.rb | 170 ------------------ 2 files changed, 175 insertions(+), 176 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 717d1627c..59c64a2ce 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -19,7 +19,7 @@ def initialize require_relative "ContainerInventoryState" require_relative "ApplicationInsightsUtility" require_relative "omslog" - require_relative "kubelet_utils" + require_relative 'CAdvisorMetricsAPIClient' end config_param :run_interval, :time, :default => 60 @@ -193,7 +193,176 @@ def inspectContainer(id, nameMap, clusterCollectEnvironmentVar) return containerInstance end - def enumerate + def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + containerInventoryRecords = Array.new + begin + response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) + if !response.nil? && !response.empty? && !response.body.nil? && !response.body.empty? + podList = JSON.parse(response.body) + if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? + podList["items"].each do |item| + containersInfoMap = getContainersInfoMap(item) + containerInventoryRecord = {} + if !item["status"].nil? && !item["status"].empty? + if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? + item["status"]["containerStatuses"].each do |containerStatus| + containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + containerName = containerStatus["name"] + containerId = containerStatus["containerID"].split('//')[1] + containerInventoryRecord["InstanceID"] = containerId + # imagedId is of the format - repo@sha256:imageid + imageIdValue = containerStatus["imageID"] + if !imageIdValue.nil? && !imageIdValue.empty? + atLocation = imageIdValue.index("@") + if !atLocation.nil? + containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] + end + end + # image is of the format - repository/image:imagetag + imageValue = containerStatus["image"] + if !imageValue.nil? && !imageValue.empty? + # Find delimiters in the string of format repository/image:imagetag + slashLocation = imageValue.index("/") + colonLocation = imageValue.index(":") + if !colonLocation.nil? + if slashLocation.nil? + # image:imagetag + containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] + else + # repository/image:imagetag + containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] + containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] + end + containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] + end + elsif !imageIdValue.nil? && !imageIdValue.empty? + # Getting repo information from imageIdValue when no tag in ImageId + if !atLocation.nil? + containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] + end + end + containerInventoryRecord["ExitCode"] = 0 + if !containerStatus["state"].nil? && !containerStatus["state"].empty? + containerState = containerStatus["state"] + if containerState.key?("running") + containerInventoryRecord["State"] = "Running" + containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] + elsif containerState.key?("terminated") + containerInventoryRecord["State"] = "Terminated" + containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] + containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] + containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] + elsif containerState.key?("waiting") + containerInventoryRecord["State"] = "Waiting" + end + end + containerInfoMap = containersInfoMap[containerName] + containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] + containerInventoryRecord["Computer"] = containerInfoMap["Computer"] + containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] + containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] + containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] + containerInventoryRecord["Ports"] = containerInfoMap["Ports"] + containerInventoryRecord["Command"] = containerInfoMap["Command"] + if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 + containerInventoryRecord["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] + else + containerInventoryRecord["EnvironmentVar"] = obtainContainerEnvironmentVars(containerId) + end + containerInventoryRecords.push containerInventoryRecord + end + end + end + end + end + end + rescue => error + @Log.warn("in_container_inventory::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") + end + return containerInventoryRecords + end + + def getContainersInfoMap(item) + containersInfoMap = {} + begin + nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" + createdTime = item["metadata"]["creationTimestamp"] + if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? + if !item["spec"]["containers"].nil? && !item["spec"]["containers"].empty? + item["spec"]["containers"].each do |container| + containerInfoMap = {} + containerName = container["name"] + containerInfoMap["ElementName"] = containerName + containerInfoMap["Computer"] = nodeName + containerInfoMap["ContainerHostname"] = nodeName + containerInfoMap["CreatedTime"] = createdTime + portsValue = container["ports"] + portsValueString = (portsValue.nil?) ? "" : portsValue.to_s + containerInfoMap["Ports"] = portsValueString + cmdValue = container["command"] + cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s + containerInfoMap["Command"] = cmdValueString + + containersInfoMap[containerName] = containerInfoMap + end + end + end + rescue => error + @Log.warn("in_container_inventory::getContainersInfoMap : Get Container Info Maps failed: #{error}") + end + return containersInfoMap + end + + def obtainContainerEnvironmentVars(containerId) + envValueString = "" + begin + unless @containerCGroupCache.has_key?(containerId) + Dir["/hostfs/proc/*/cgroup"].each do| filename| + if File.file?(filename) && File.foreach(filename).grep(/#{containerId}/).any? + # file full path is /hostfs/proc//cgroup + cGroupPid = filename.split("/")[3] + if @containerCGroupCache.has_key?(containerId) + tempCGroupPid = containerCgroupCache[containerId] + if tempCGroupPid > cGroupPid + @containerCgroupCache[containerId] = cGroupPid + end + else + @containerCGroupCache[containerId] = cGroupPid + end + end + end + end + cGroupPid = @containerCGroupCache[containerId] + if !cGroupPid.nil? + environFilePath = "/hostfs/proc/#{cGroupPid}/environ" + if File.exist?(environFilePath) + # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE + # Check to see if the environment variable collection is disabled for this container. + if File.foreach(environFilePath).grep(/AZMON_COLLECT_ENV=FALSE/i).any? + envValueString = ["AZMON_COLLECT_ENV=FALSE"] + $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") + else + fileSize = File.size(environFilePath) + # Restricting the ENV string value to 200kb since the size of this string can go very high + envVars = File.read(environFilePath, 200000).split(" ") + envValueString = envVars.to_s + if fileSize > 200000 + lastIndex = envValueString.rindex("\", ") + if !lastIndex.nil? + envValueStringTruncated = envValueString.slice(0..lastIndex) + "]" + envValueString = envValueStringTruncated + end + end + end + end + end + rescue => error + @Log.warn("in_container_inventory::obtainContainerEnvironmentVars : obtain Container Environment vars failed: #{error} for containerId: #{containerId}") + end + return envValueString + end + + def enumerate currentTime = Time.now emitTime = currentTime.to_f batchTime = currentTime.utc.iso8601 @@ -236,10 +405,10 @@ def enumerate end end else - $log.info("in_container_inventory::enumerate : using cadvisor apis since container runtime is non docker") - containerInventoryRecords = KubeletUtils.getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar, @containerCGroupCache) + $log.info("in_container_inventory::enumerate : using cadvisor apis since non docker container runtime : #{containerRuntimeEnv}") + containerInventoryRecords = getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerIds = Array.new - containerInventoryRecords.each do |containerRecord| + containerInventoryRecords.each do |containerRecord| ContainerInventoryState.writeContainerState(containerRecord) if hostName.empty? && !containerRecord["Computer"].empty? hostName = containerRecord["Computer"] @@ -255,7 +424,7 @@ def enumerate if !container.nil? container.each { |k, v| container[k] = v } container["State"] = "Deleted" - @containerCGroupCache.delete(container["InstanceID"]) + @containerCGroupCache.delete(container["InstanceID"]) containerInventory.push container end end diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 54accb512..fecb339b9 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -19,175 +19,5 @@ def get_node_capacity return [cpu_capacity, memory_capacity] end end - - def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar, @containerCGroupCache) - containerInventoryRecords = Array.new - begin - response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) - if !response.nil? && !response.body.nil? - podList = JSON.parse(response.body) - if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? - podList["items"].each do |item| - containersInfoMap = getContainersInfoMap(item) - containerInventoryRecord = {} - if !item["status"].nil? && !item["status"].empty? - if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? - item["status"]["containerStatuses"].each do |containerStatus| - containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated - containerName = containerStatus["name"] - containerId = containerStatus["containerID"].split('//')[1] - containerInventoryRecord["InstanceID"] = containerId - # imagedId is of the format - repo@sha256:imageid - imageIdValue = containerStatus["imageID"] - if !imageIdValue.nil? && !imageIdValue.empty? - atLocation = imageIdValue.index("@") - if !atLocation.nil? - containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] - end - end - # image is of the format - repository/image:imagetag - imageValue = containerStatus["image"] - if !imageValue.nil? && !imageValue.empty? - # Find delimiters in the string of format repository/image:imagetag - slashLocation = imageValue.index("/") - colonLocation = imageValue.index(":") - if !colonLocation.nil? - if slashLocation.nil? - # image:imagetag - containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] - else - # repository/image:imagetag - containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] - containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] - end - containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] - end - elsif !imageIdValue.nil? && !imageIdValue.empty? - # Getting repo information from imageIdValue when no tag in ImageId - if !atLocation.nil? - containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] - end - end - containerInventoryRecord["ExitCode"] = 0 - if !containerStatus["state"].nil? && !containerStatus["state"].empty? - containerState = containerStatus["state"] - if containerState.key?("running") - containerInventoryRecord["State"] = "Running" - containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] - elsif containerState.key?("terminated") - containerInventoryRecord["State"] = "Terminated" - containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] - containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] - containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] - elsif containerState.key?("waiting") - containerInventoryRecord["State"] = "Waiting" - end - end - containerInfoMap = containersInfoMap[containerName] - containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] - containerInventoryRecord["Computer"] = containerInfoMap["Computer"] - containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] - containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] - containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] - containerInventoryRecord["Ports"] = containerInfoMap["Ports"] - containerInventoryRecord["Command"] = containerInfoMap["Command"] - if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 - containerInventoryRecord["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] - else - containerInventoryRecord["EnvironmentVar"] = KubeletUtils.obtainContainerEnvironmentVars(containerId, @containerCGroupCache) - end - containerInventoryRecords.push containerInventoryRecord - end - end - end - end - end - end - rescue => error - @Log.warn("kubelet_utils::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") - end - return containerInventoryRecords - end - - def getContainersInfoMap(item) - containersInfoMap = {} - begin - nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" - createdTime = item["metadata"]["creationTimestamp"] - if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? - if !item["spec"]["containers"].nil? && !item["spec"]["containers"].empty? - item["spec"]["containers"].each do |container| - containerInfoMap = {} - containerName = container["name"] - containerInfoMap["ElementName"] = containerName - containerInfoMap["Computer"] = nodeName - containerInfoMap["ContainerHostname"] = nodeName - containerInfoMap["CreatedTime"] = createdTime - portsValue = container["ports"] - portsValueString = (portsValue.nil?) ? "" : portsValue.to_s - containerInfoMap["Ports"] = portsValueString - cmdValue = container["command"] - cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s - containerInfoMap["Command"] = cmdValueString - - containersInfoMap[containerName] = containerInfoMap - end - end - end - rescue => error - @Log.warn("kubelet_utils::getContainersInfoMap : Get Container Info Maps failed: #{error}") - end - return containersInfoMap - end - - def obtainContainerEnvironmentVars(containerId, containerCGroupCache) - envValueString = "" - begin - unless containerCGroupCache.has_key?(containerId) - Dir["/hostfs/proc/*/cgroup"].each do| filename| - if File.file?(filename) && File.foreach(filename).grep(/#{containerId}/).any? - # file full path is /hostfs/proc//cgroup - cGroupPid = filename.split("/")[3] - if containerCGroupCache.has_key?(containerId) - temCGroupPid = containerCgroupCache[containerId] - if temCGroupPid > cGroupPid - containerCgroupCache[containerId] = cGroupPid - end - else - containerCGroupCache[containerId] = cGroupPid - end - end - end - end - cGroupPid = containerCGroupCache[containerId] - if !cGroupPid.nil? - environFilePath = "/hostfs/proc/#{cGroupPid}/environ" - if File.exist?(environFilePath) - # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE - # Check to see if the environment variable collection is disabled for this container. - if File.foreach(environFilePath).grep(/AZMON_COLLECT_ENV=FALSE/i).any? - envValueString = ["AZMON_COLLECT_ENV=FALSE"] - $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") - else - fileSize = File.size(environFilePath) - # Restricting the ENV string value to 200kb since the size of this string can go very high - envVars = File.read(environFilePath, 200000).split(" ") - envValueString = envVars.to_s - if fileSize > 200000 - lastIndex = envValueString.rindex("\", ") - if !lastIndex.nil? - envValueStringTruncated = envValueString.slice(0..lastIndex) + "]" - envValueString = envValueStringTruncated - end - end - end - end - end - rescue => error - @Log.warn("kubelet_utils::obtainContainerEnvironmentVars : obtain Container Environment vars failed: #{error} for containerId: #{containerId}") - end - return envValueString - end - end end \ No newline at end of file From a704ac46deb310928039be5ff414c88d8d324434 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 20 Feb 2020 12:43:20 -0800 Subject: [PATCH 28/73] fix log warn --- source/code/plugin/in_containerinventory.rb | 70 ++++++++++----------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 59c64a2ce..53a6b6e20 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -210,37 +210,37 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerName = containerStatus["name"] containerId = containerStatus["containerID"].split('//')[1] containerInventoryRecord["InstanceID"] = containerId - # imagedId is of the format - repo@sha256:imageid - imageIdValue = containerStatus["imageID"] - if !imageIdValue.nil? && !imageIdValue.empty? - atLocation = imageIdValue.index("@") - if !atLocation.nil? - containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] - end - end - # image is of the format - repository/image:imagetag - imageValue = containerStatus["image"] - if !imageValue.nil? && !imageValue.empty? - # Find delimiters in the string of format repository/image:imagetag - slashLocation = imageValue.index("/") - colonLocation = imageValue.index(":") - if !colonLocation.nil? - if slashLocation.nil? - # image:imagetag - containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] - else - # repository/image:imagetag - containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] - containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] + # imagedId is of the format - repo@sha256:imageid + imageIdValue = containerStatus["imageID"] + if !imageIdValue.nil? && !imageIdValue.empty? + atLocation = imageIdValue.index("@") + if !atLocation.nil? + containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] end - containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] - end - elsif !imageIdValue.nil? && !imageIdValue.empty? - # Getting repo information from imageIdValue when no tag in ImageId - if !atLocation.nil? - containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] - end - end + end + # image is of the format - repository/image:imagetag + imageValue = containerStatus["image"] + if !imageValue.nil? && !imageValue.empty? + # Find delimiters in the string of format repository/image:imagetag + slashLocation = imageValue.index("/") + colonLocation = imageValue.index(":") + if !colonLocation.nil? + if slashLocation.nil? + # image:imagetag + containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] + else + # repository/image:imagetag + containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] + containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] + end + containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] + end + elsif !imageIdValue.nil? && !imageIdValue.empty? + # Getting repo information from imageIdValue when no tag in ImageId + if !atLocation.nil? + containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] + end + end containerInventoryRecord["ExitCode"] = 0 if !containerStatus["state"].nil? && !containerStatus["state"].empty? containerState = containerStatus["state"] @@ -265,9 +265,9 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerInventoryRecord["Ports"] = containerInfoMap["Ports"] containerInventoryRecord["Command"] = containerInfoMap["Command"] if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 - containerInventoryRecord["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] + containerInventoryRecord["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] else - containerInventoryRecord["EnvironmentVar"] = obtainContainerEnvironmentVars(containerId) + containerInventoryRecord["EnvironmentVar"] = obtainContainerEnvironmentVars(containerId) end containerInventoryRecords.push containerInventoryRecord end @@ -277,7 +277,7 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) end end rescue => error - @Log.warn("in_container_inventory::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") + @log.warn("in_container_inventory::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") end return containerInventoryRecords end @@ -308,7 +308,7 @@ def getContainersInfoMap(item) end end rescue => error - @Log.warn("in_container_inventory::getContainersInfoMap : Get Container Info Maps failed: #{error}") + @log.warn("in_container_inventory::getContainersInfoMap : Get Container Info Maps failed: #{error}") end return containersInfoMap end @@ -357,7 +357,7 @@ def obtainContainerEnvironmentVars(containerId) end end rescue => error - @Log.warn("in_container_inventory::obtainContainerEnvironmentVars : obtain Container Environment vars failed: #{error} for containerId: #{containerId}") + @log.warn("in_container_inventory::obtainContainerEnvironmentVars : obtain Container Environment vars failed: #{error} for containerId: #{containerId}") end return envValueString end From e0905d3b196323f85958fee20e2d15b8bcbbb225 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 20 Feb 2020 12:53:09 -0800 Subject: [PATCH 29/73] add more logging --- source/code/plugin/in_containerinventory.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 53a6b6e20..d593d7405 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -314,12 +314,15 @@ def getContainersInfoMap(item) end def obtainContainerEnvironmentVars(containerId) + $log.info("in_container_inventory::obtainContainerEnvironmentVars @ #{Time.now.utc.iso8601}") envValueString = "" begin unless @containerCGroupCache.has_key?(containerId) + $log.info("in_container_inventory::fetching cGroup parent pid @ #{Time.now.utc.iso8601}") Dir["/hostfs/proc/*/cgroup"].each do| filename| if File.file?(filename) && File.foreach(filename).grep(/#{containerId}/).any? # file full path is /hostfs/proc//cgroup + $log.info("in_container_inventory::fetching cGroup parent filename @ #{filename}") cGroupPid = filename.split("/")[3] if @containerCGroupCache.has_key?(containerId) tempCGroupPid = containerCgroupCache[containerId] @@ -343,6 +346,7 @@ def obtainContainerEnvironmentVars(containerId) $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") else fileSize = File.size(environFilePath) + $log.info("in_container_inventory::environment vars filename @ #{filename} filesize @ #{fileSize}") # Restricting the ENV string value to 200kb since the size of this string can go very high envVars = File.read(environFilePath, 200000).split(" ") envValueString = envVars.to_s @@ -411,14 +415,14 @@ def enumerate containerInventoryRecords.each do |containerRecord| ContainerInventoryState.writeContainerState(containerRecord) if hostName.empty? && !containerRecord["Computer"].empty? - hostName = containerRecord["Computer"] + hostName = containerRecord["Computer"] end containerIds.push containerRecord["InstanceID"] containerInventory.push containerRecord end # Update the state for deleted containers deletedContainers = ContainerInventoryState.getDeletedContainers(containerIds) - if !deletedContainers.nil? && !deletedContainers.empty? + if !deletedContainers.nil? && !deletedContainers.empty? deletedContainers.each do |deletedContainer| container = ContainerInventoryState.readContainerState(deletedContainer) if !container.nil? @@ -428,7 +432,7 @@ def enumerate containerInventory.push container end end - end + end end containerInventory.each do |record| From 264aa314046aaf053f37e23de5af5b21d3a8afff Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 20 Feb 2020 13:52:00 -0800 Subject: [PATCH 30/73] fix npe --- source/code/plugin/in_containerinventory.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index d593d7405..dc3cc04b6 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -197,7 +197,7 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerInventoryRecords = Array.new begin response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) - if !response.nil? && !response.empty? && !response.body.nil? && !response.body.empty? + if !response.nil? && !response.body.nil? podList = JSON.parse(response.body) if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? podList["items"].each do |item| From c218c195272a5faf2bf9594203cf27b20562b100 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Thu, 20 Feb 2020 22:47:15 -0800 Subject: [PATCH 31/73] revert kubelet api to get the envvars --- source/code/plugin/in_containerinventory.rb | 188 +------------------- source/code/plugin/kubelet_utils.rb | 179 +++++++++++++++++++ 2 files changed, 185 insertions(+), 182 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index dc3cc04b6..b5b8cee5f 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -18,8 +18,8 @@ def initialize require_relative "DockerApiClient" require_relative "ContainerInventoryState" require_relative "ApplicationInsightsUtility" - require_relative "omslog" - require_relative 'CAdvisorMetricsAPIClient' + require_relative "omslog" + require_relative "kubelet_utils" end config_param :run_interval, :time, :default => 60 @@ -35,9 +35,7 @@ def start @condition = ConditionVariable.new @mutex = Mutex.new @thread = Thread.new(&method(:run_periodic)) - @@telemetryTimeTracker = DateTime.now.to_time.to_i - # cache the container and cgroup parent process - @containerCGroupCache = Hash.new + @@telemetryTimeTracker = DateTime.now.to_time.to_i end end @@ -191,180 +189,7 @@ def inspectContainer(id, nameMap, clusterCollectEnvironmentVar) $log.warn("Exception in inspectContainer: #{errorStr} for container: #{id}") end return containerInstance - end - - def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) - containerInventoryRecords = Array.new - begin - response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) - if !response.nil? && !response.body.nil? - podList = JSON.parse(response.body) - if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? - podList["items"].each do |item| - containersInfoMap = getContainersInfoMap(item) - containerInventoryRecord = {} - if !item["status"].nil? && !item["status"].empty? - if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? - item["status"]["containerStatuses"].each do |containerStatus| - containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated - containerName = containerStatus["name"] - containerId = containerStatus["containerID"].split('//')[1] - containerInventoryRecord["InstanceID"] = containerId - # imagedId is of the format - repo@sha256:imageid - imageIdValue = containerStatus["imageID"] - if !imageIdValue.nil? && !imageIdValue.empty? - atLocation = imageIdValue.index("@") - if !atLocation.nil? - containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] - end - end - # image is of the format - repository/image:imagetag - imageValue = containerStatus["image"] - if !imageValue.nil? && !imageValue.empty? - # Find delimiters in the string of format repository/image:imagetag - slashLocation = imageValue.index("/") - colonLocation = imageValue.index(":") - if !colonLocation.nil? - if slashLocation.nil? - # image:imagetag - containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] - else - # repository/image:imagetag - containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] - containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] - end - containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] - end - elsif !imageIdValue.nil? && !imageIdValue.empty? - # Getting repo information from imageIdValue when no tag in ImageId - if !atLocation.nil? - containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] - end - end - containerInventoryRecord["ExitCode"] = 0 - if !containerStatus["state"].nil? && !containerStatus["state"].empty? - containerState = containerStatus["state"] - if containerState.key?("running") - containerInventoryRecord["State"] = "Running" - containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] - elsif containerState.key?("terminated") - containerInventoryRecord["State"] = "Terminated" - containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] - containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] - containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] - elsif containerState.key?("waiting") - containerInventoryRecord["State"] = "Waiting" - end - end - containerInfoMap = containersInfoMap[containerName] - containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] - containerInventoryRecord["Computer"] = containerInfoMap["Computer"] - containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] - containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] - containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] - containerInventoryRecord["Ports"] = containerInfoMap["Ports"] - containerInventoryRecord["Command"] = containerInfoMap["Command"] - if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 - containerInventoryRecord["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] - else - containerInventoryRecord["EnvironmentVar"] = obtainContainerEnvironmentVars(containerId) - end - containerInventoryRecords.push containerInventoryRecord - end - end - end - end - end - end - rescue => error - @log.warn("in_container_inventory::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") - end - return containerInventoryRecords - end - - def getContainersInfoMap(item) - containersInfoMap = {} - begin - nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" - createdTime = item["metadata"]["creationTimestamp"] - if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? - if !item["spec"]["containers"].nil? && !item["spec"]["containers"].empty? - item["spec"]["containers"].each do |container| - containerInfoMap = {} - containerName = container["name"] - containerInfoMap["ElementName"] = containerName - containerInfoMap["Computer"] = nodeName - containerInfoMap["ContainerHostname"] = nodeName - containerInfoMap["CreatedTime"] = createdTime - portsValue = container["ports"] - portsValueString = (portsValue.nil?) ? "" : portsValue.to_s - containerInfoMap["Ports"] = portsValueString - cmdValue = container["command"] - cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s - containerInfoMap["Command"] = cmdValueString - - containersInfoMap[containerName] = containerInfoMap - end - end - end - rescue => error - @log.warn("in_container_inventory::getContainersInfoMap : Get Container Info Maps failed: #{error}") - end - return containersInfoMap - end - - def obtainContainerEnvironmentVars(containerId) - $log.info("in_container_inventory::obtainContainerEnvironmentVars @ #{Time.now.utc.iso8601}") - envValueString = "" - begin - unless @containerCGroupCache.has_key?(containerId) - $log.info("in_container_inventory::fetching cGroup parent pid @ #{Time.now.utc.iso8601}") - Dir["/hostfs/proc/*/cgroup"].each do| filename| - if File.file?(filename) && File.foreach(filename).grep(/#{containerId}/).any? - # file full path is /hostfs/proc//cgroup - $log.info("in_container_inventory::fetching cGroup parent filename @ #{filename}") - cGroupPid = filename.split("/")[3] - if @containerCGroupCache.has_key?(containerId) - tempCGroupPid = containerCgroupCache[containerId] - if tempCGroupPid > cGroupPid - @containerCgroupCache[containerId] = cGroupPid - end - else - @containerCGroupCache[containerId] = cGroupPid - end - end - end - end - cGroupPid = @containerCGroupCache[containerId] - if !cGroupPid.nil? - environFilePath = "/hostfs/proc/#{cGroupPid}/environ" - if File.exist?(environFilePath) - # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE - # Check to see if the environment variable collection is disabled for this container. - if File.foreach(environFilePath).grep(/AZMON_COLLECT_ENV=FALSE/i).any? - envValueString = ["AZMON_COLLECT_ENV=FALSE"] - $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") - else - fileSize = File.size(environFilePath) - $log.info("in_container_inventory::environment vars filename @ #{filename} filesize @ #{fileSize}") - # Restricting the ENV string value to 200kb since the size of this string can go very high - envVars = File.read(environFilePath, 200000).split(" ") - envValueString = envVars.to_s - if fileSize > 200000 - lastIndex = envValueString.rindex("\", ") - if !lastIndex.nil? - envValueStringTruncated = envValueString.slice(0..lastIndex) + "]" - envValueString = envValueStringTruncated - end - end - end - end - end - rescue => error - @log.warn("in_container_inventory::obtainContainerEnvironmentVars : obtain Container Environment vars failed: #{error} for containerId: #{containerId}") - end - return envValueString - end + end def enumerate currentTime = Time.now @@ -410,7 +235,7 @@ def enumerate end else $log.info("in_container_inventory::enumerate : using cadvisor apis since non docker container runtime : #{containerRuntimeEnv}") - containerInventoryRecords = getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + containerInventoryRecords = KubeletUtils.getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerIds = Array.new containerInventoryRecords.each do |containerRecord| ContainerInventoryState.writeContainerState(containerRecord) @@ -427,8 +252,7 @@ def enumerate container = ContainerInventoryState.readContainerState(deletedContainer) if !container.nil? container.each { |k, v| container[k] = v } - container["State"] = "Deleted" - @containerCGroupCache.delete(container["InstanceID"]) + container["State"] = "Deleted" containerInventory.push container end end diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index fecb339b9..0fa925301 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -19,5 +19,184 @@ def get_node_capacity return [cpu_capacity, memory_capacity] end end + + def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + containerInventoryRecords = Array.new + begin + response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) + if !response.nil? && !response.body.nil? + podList = JSON.parse(response.body) + if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? + podList["items"].each do |item| + containersInfoMap = getContainersInfoMap(item, clusterCollectEnvironmentVar) + containerInventoryRecord = {} + if !item["status"].nil? && !item["status"].empty? + if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? + item["status"]["containerStatuses"].each do |containerStatus| + containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + containerName = containerStatus["name"] + # containeId format is :// + containerId = containerStatus["containerID"].split('//')[1] + containerInventoryRecord["InstanceID"] = containerId + # imagedId is of the format - repo@sha256:imageid + imageIdValue = containerStatus["imageID"] + if !imageIdValue.nil? && !imageIdValue.empty? + atLocation = imageIdValue.index("@") + if !atLocation.nil? + containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] + end + end + # image is of the format - repository/image:imagetag + imageValue = containerStatus["image"] + if !imageValue.nil? && !imageValue.empty? + # Find delimiters in the string of format repository/image:imagetag + slashLocation = imageValue.index("/") + colonLocation = imageValue.index(":") + if !colonLocation.nil? + if slashLocation.nil? + # image:imagetag + containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] + else + # repository/image:imagetag + containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] + containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] + end + containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] + end + elsif !imageIdValue.nil? && !imageIdValue.empty? + # Getting repo information from imageIdValue when no tag in ImageId + if !atLocation.nil? + containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] + end + end + containerInventoryRecord["ExitCode"] = 0 + if !containerStatus["state"].nil? && !containerStatus["state"].empty? + containerState = containerStatus["state"] + if containerState.key?("running") + containerInventoryRecord["State"] = "Running" + containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] + elsif containerState.key?("terminated") + containerInventoryRecord["State"] = "Terminated" + containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] + containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] + containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] + elsif containerState.key?("waiting") + containerInventoryRecord["State"] = "Waiting" + end + end + containerInfoMap = containersInfoMap[containerName] + containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] + containerInventoryRecord["Computer"] = containerInfoMap["Computer"] + containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] + containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] + containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] + containerInventoryRecord["Ports"] = containerInfoMap["Ports"] + containerInventoryRecord["Command"] = containerInfoMap["Command"] + containerInventoryRecords.push containerInventoryRecord + end + end + end + end + end + end + rescue => error + @log.warn("KubeletUtils::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") + end + return containerInventoryRecords + end + + def getContainersInfoMap(item, clusterCollectEnvironmentVar) + containersInfoMap = {} + begin + nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" + createdTime = item["metadata"]["creationTimestamp"] + if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? + if !item["spec"]["containers"].nil? && !item["spec"]["containers"].empty? + item["spec"]["containers"].each do |container| + containerInfoMap = {} + containerName = container["name"] + containerInfoMap["ElementName"] = containerName + containerInfoMap["Computer"] = nodeName + containerInfoMap["ContainerHostname"] = nodeName + containerInfoMap["CreatedTime"] = createdTime + if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 + containerInfoMap["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] + else + containerInfoMap["EnvironmentVar"] = obtainContainerEnvironmentVariable(container["env"]) + end + portsValue = container["ports"] + portsValueString = (portsValue.nil?) ? "" : portsValue.to_s + containerInfoMap["Ports"] = portsValueString + cmdValue = container["command"] + cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s + containerInfoMap["Command"] = cmdValueString + + containersInfoMap[containerName] = containerInfoMap + end + end + end + rescue => error + @log.warn("KubeletUtils::getContainersInfoMap : Get Container Info Maps failed: #{error}") + end + return containersInfoMap + end + + def obtainContainerEnvironmentVariable(envVarsJSON) + envValueString = "" + begin + envVars = [] + if !envVarsJSON.nil? && !envVarsJSON.empty? + envVarsJSON.each do |envVar| + key = envVar["name"] + value = "" + if !envVar["value"].nil? + value = envVar["value"] + elsif !envVar["valueFrom"].nil? + valueFrom = envVar["valueFrom"] + if valueFrom.key?("fieldRef") && !valueFrom["fieldRef"]["fieldPath"].nil? && !valueFrom["fieldRef"]["fieldPath"].empty? + fieldPath = valueFrom["fieldRef"]["fieldPath"] + fields = fieldPath.split('.') + if fields.length() == 2 + # handle fieldRef of labels and annotations + if !fields[1].nil? && !fields[1].empty? & fields[1].end_with?(']') + indexFields = fields[1].split('[') + hashMapValue = item[fields[0]][indexFields[0]] + if !hashMapValue.nil? && !hashMapValue.empty? + subField = indexFields[1].delete_suffix("]").delete("\\'") + value = hashMapValue[subField] + end + else + value = item[fields[0]][fields[1]] + end + end + end + else + value = envVar["valueFrom"].to_s + end + envVars.push("#{key}=#{value}") + end + envValueString = envVars.to_s + # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE + # Check to see if the environment variable collection is disabled for this container. + if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) + envValueString = ["AZMON_COLLECT_ENV=FALSE"] + $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") + else + # Restricting the ENV string value to 200kb since the size of this string can go very high + if envValueString.length > 200000 + envValueStringTruncated = envValueString.slice(0..200000) + lastIndex = envValueStringTruncated.rindex("\", ") + if !lastIndex.nil? + envValueString = envValueStringTruncated.slice(0..lastIndex) + "]" + end + envValueString = envValueStringTruncated + end + end + end + rescue => error + @log.warn("in_container_inventory::getContainersInfoMap : Get Container Info Maps failed: #{error}") + end + return envValueString + end end end \ No newline at end of file From 977b0fe6540892fbbee8d91d65a9223e5caf50aa Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 21 Feb 2020 00:12:12 -0800 Subject: [PATCH 32/73] fix minor issue --- source/code/plugin/kubelet_utils.rb | 34 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 0fa925301..61c44ca60 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -157,23 +157,22 @@ def obtainContainerEnvironmentVariable(envVarsJSON) fieldPath = valueFrom["fieldRef"]["fieldPath"] fields = fieldPath.split('.') if fields.length() == 2 - # handle fieldRef of labels and annotations - if !fields[1].nil? && !fields[1].empty? & fields[1].end_with?(']') - indexFields = fields[1].split('[') - hashMapValue = item[fields[0]][indexFields[0]] - if !hashMapValue.nil? && !hashMapValue.empty? - subField = indexFields[1].delete_suffix("]").delete("\\'") - value = hashMapValue[subField] - end + if !fields[1].nil? && !fields[1].empty? & fields[1].end_with?(']') + indexFields = fields[1].split('[') + hashMapValue = item[fields[0]][indexFields[0]] + if !hashMapValue.nil? && !hashMapValue.empty? + subField = indexFields[1].delete_suffix("]").delete("\\'") + value = hashMapValue[subField] + end else - value = item[fields[0]][fields[1]] + value = item[fields[0]][fields[1]] end - end - end - else - value = envVar["valueFrom"].to_s - end - envVars.push("#{key}=#{value}") + end + else + value = envVar["valueFrom"].to_s + end + end + envVars.push("#{key}=#{value}") end envValueString = envVars.to_s # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE @@ -188,8 +187,9 @@ def obtainContainerEnvironmentVariable(envVarsJSON) lastIndex = envValueStringTruncated.rindex("\", ") if !lastIndex.nil? envValueString = envValueStringTruncated.slice(0..lastIndex) + "]" - end - envValueString = envValueStringTruncated + else + envValueString = envValueStringTruncated + end end end end From e785c4af5f5da31933966320499af918d9609f2c Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 21 Feb 2020 08:01:57 -0800 Subject: [PATCH 33/73] fix log message --- source/code/plugin/kubelet_utils.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 61c44ca60..63f04d117 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -122,7 +122,7 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 containerInfoMap["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] else - containerInfoMap["EnvironmentVar"] = obtainContainerEnvironmentVariable(container["env"]) + containerInfoMap["EnvironmentVar"] = parseContainerEnvironmentVarsJSON(containerName, container["env"]) end portsValue = container["ports"] portsValueString = (portsValue.nil?) ? "" : portsValue.to_s @@ -141,7 +141,7 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) return containersInfoMap end - def obtainContainerEnvironmentVariable(envVarsJSON) + def parseContainerEnvironmentVarsJSON(containerName, envVarsJSON) envValueString = "" begin envVars = [] @@ -194,7 +194,7 @@ def obtainContainerEnvironmentVariable(envVarsJSON) end end rescue => error - @log.warn("in_container_inventory::getContainersInfoMap : Get Container Info Maps failed: #{error}") + @log.warn("KubeletUtils::parseContainerEnvironmentVarsJSON : parsing of EnvVars JSON failed: #{error}") end return envValueString end From f0a5f2c84077a8ed696f1df76e59ea239763064c Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 21 Feb 2020 10:03:17 -0800 Subject: [PATCH 34/73] comment valueFrom for now since this causing some issue --- source/code/plugin/kubelet_utils.rb | 66 ++++++++++++++--------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 63f04d117..17b679939 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -91,7 +91,7 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] containerInventoryRecord["Ports"] = containerInfoMap["Ports"] - containerInventoryRecord["Command"] = containerInfoMap["Command"] + containerInventoryRecord["Command"] = containerInfoMap["Command"] containerInventoryRecords.push containerInventoryRecord end end @@ -104,7 +104,7 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) end return containerInventoryRecords end - + def getContainersInfoMap(item, clusterCollectEnvironmentVar) containersInfoMap = {} begin @@ -121,16 +121,16 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) containerInfoMap["CreatedTime"] = createdTime if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 containerInfoMap["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] - else + else containerInfoMap["EnvironmentVar"] = parseContainerEnvironmentVarsJSON(containerName, container["env"]) - end + end portsValue = container["ports"] portsValueString = (portsValue.nil?) ? "" : portsValue.to_s containerInfoMap["Ports"] = portsValueString cmdValue = container["command"] cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s containerInfoMap["Command"] = cmdValueString - + containersInfoMap[containerName] = containerInfoMap end end @@ -151,52 +151,52 @@ def parseContainerEnvironmentVarsJSON(containerName, envVarsJSON) value = "" if !envVar["value"].nil? value = envVar["value"] - elsif !envVar["valueFrom"].nil? - valueFrom = envVar["valueFrom"] - if valueFrom.key?("fieldRef") && !valueFrom["fieldRef"]["fieldPath"].nil? && !valueFrom["fieldRef"]["fieldPath"].empty? - fieldPath = valueFrom["fieldRef"]["fieldPath"] - fields = fieldPath.split('.') - if fields.length() == 2 - if !fields[1].nil? && !fields[1].empty? & fields[1].end_with?(']') - indexFields = fields[1].split('[') - hashMapValue = item[fields[0]][indexFields[0]] - if !hashMapValue.nil? && !hashMapValue.empty? - subField = indexFields[1].delete_suffix("]").delete("\\'") - value = hashMapValue[subField] - end - else - value = item[fields[0]][fields[1]] - end - end - else - value = envVar["valueFrom"].to_s - end - end - envVars.push("#{key}=#{value}") - end + # elsif !envVar["valueFrom"].nil? + # valueFrom = envVar["valueFrom"] + # if valueFrom.key?("fieldRef") && !valueFrom["fieldRef"]["fieldPath"].nil? && !valueFrom["fieldRef"]["fieldPath"].empty? + # fieldPath = valueFrom["fieldRef"]["fieldPath"] + # fields = fieldPath.split('.') + # if fields.length() == 2 + # if !fields[1].nil? && !fields[1].empty? & fields[1].end_with?(']') + # indexFields = fields[1].split('[') + # hashMapValue = item[fields[0]][indexFields[0]] + # if !hashMapValue.nil? && !hashMapValue.empty? + # subField = indexFields[1].delete_suffix("]").delete("\\'") + # value = hashMapValue[subField] + # end + # else + # value = item[fields[0]][fields[1]] + # end + # end + # else + # value = envVar["valueFrom"].to_s + # end + end + envVars.push("#{key}=#{value}") + end envValueString = envVars.to_s # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE # Check to see if the environment variable collection is disabled for this container. if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) envValueString = ["AZMON_COLLECT_ENV=FALSE"] $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") - else + else # Restricting the ENV string value to 200kb since the size of this string can go very high if envValueString.length > 200000 envValueStringTruncated = envValueString.slice(0..200000) lastIndex = envValueStringTruncated.rindex("\", ") if !lastIndex.nil? envValueString = envValueStringTruncated.slice(0..lastIndex) + "]" - else + else envValueString = envValueStringTruncated end end - end - end + end + end rescue => error @log.warn("KubeletUtils::parseContainerEnvironmentVarsJSON : parsing of EnvVars JSON failed: #{error}") end return envValueString - end + end end end \ No newline at end of file From 7d40af1f4b1eb55cd588cfc6fe1467e1caa2c2ec Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 21 Feb 2020 18:29:12 -0800 Subject: [PATCH 35/73] fix formatting issue --- source/code/plugin/kubelet_utils.rb | 357 ++++++++++++++-------------- 1 file changed, 178 insertions(+), 179 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 17b679939..870729028 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -2,201 +2,200 @@ #!/usr/local/bin/ruby # frozen_string_literal: true -require_relative 'CAdvisorMetricsAPIClient' +require_relative "CAdvisorMetricsAPIClient" class KubeletUtils - class << self - def get_node_capacity + class << self + def get_node_capacity + cpu_capacity = 1.0 + memory_capacity = 1.0 - cpu_capacity = 1.0 - memory_capacity = 1.0 - - response = CAdvisorMetricsAPIClient.getNodeCapacityFromCAdvisor(winNode: nil) - if !response.nil? && !response.body.nil? - cpu_capacity = JSON.parse(response.body)["num_cores"].nil? ? 1.0 : (JSON.parse(response.body)["num_cores"] * 1000.0) - memory_capacity = JSON.parse(response.body)["memory_capacity"].nil? ? 1.0 : JSON.parse(response.body)["memory_capacity"].to_f - $log.info "CPU = #{cpu_capacity}mc Memory = #{memory_capacity/1024/1024}MB" - return [cpu_capacity, memory_capacity] - end - end + response = CAdvisorMetricsAPIClient.getNodeCapacityFromCAdvisor(winNode: nil) + if !response.nil? && !response.body.nil? + cpu_capacity = JSON.parse(response.body)["num_cores"].nil? ? 1.0 : (JSON.parse(response.body)["num_cores"] * 1000.0) + memory_capacity = JSON.parse(response.body)["memory_capacity"].nil? ? 1.0 : JSON.parse(response.body)["memory_capacity"].to_f + $log.info "CPU = #{cpu_capacity}mc Memory = #{memory_capacity / 1024 / 1024}MB" + return [cpu_capacity, memory_capacity] + end + end - def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) - containerInventoryRecords = Array.new - begin - response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) - if !response.nil? && !response.body.nil? - podList = JSON.parse(response.body) - if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? - podList["items"].each do |item| - containersInfoMap = getContainersInfoMap(item, clusterCollectEnvironmentVar) - containerInventoryRecord = {} - if !item["status"].nil? && !item["status"].empty? - if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? - item["status"]["containerStatuses"].each do |containerStatus| - containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated - containerName = containerStatus["name"] - # containeId format is :// - containerId = containerStatus["containerID"].split('//')[1] - containerInventoryRecord["InstanceID"] = containerId - # imagedId is of the format - repo@sha256:imageid - imageIdValue = containerStatus["imageID"] - if !imageIdValue.nil? && !imageIdValue.empty? - atLocation = imageIdValue.index("@") - if !atLocation.nil? - containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] - end - end - # image is of the format - repository/image:imagetag - imageValue = containerStatus["image"] - if !imageValue.nil? && !imageValue.empty? - # Find delimiters in the string of format repository/image:imagetag - slashLocation = imageValue.index("/") - colonLocation = imageValue.index(":") - if !colonLocation.nil? - if slashLocation.nil? - # image:imagetag - containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] - else - # repository/image:imagetag - containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] - containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] - end - containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] - end - elsif !imageIdValue.nil? && !imageIdValue.empty? - # Getting repo information from imageIdValue when no tag in ImageId - if !atLocation.nil? - containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] - end - end - containerInventoryRecord["ExitCode"] = 0 - if !containerStatus["state"].nil? && !containerStatus["state"].empty? - containerState = containerStatus["state"] - if containerState.key?("running") - containerInventoryRecord["State"] = "Running" - containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] - elsif containerState.key?("terminated") - containerInventoryRecord["State"] = "Terminated" - containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] - containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] - containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] - elsif containerState.key?("waiting") - containerInventoryRecord["State"] = "Waiting" - end - end - containerInfoMap = containersInfoMap[containerName] - containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] - containerInventoryRecord["Computer"] = containerInfoMap["Computer"] - containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] - containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] - containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] - containerInventoryRecord["Ports"] = containerInfoMap["Ports"] - containerInventoryRecord["Command"] = containerInfoMap["Command"] - containerInventoryRecords.push containerInventoryRecord - end - end - end + def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + containerInventoryRecords = Array.new + begin + response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) + if !response.nil? && !response.body.nil? + podList = JSON.parse(response.body) + if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? + podList["items"].each do |item| + containersInfoMap = getContainersInfoMap(item, clusterCollectEnvironmentVar) + containerInventoryRecord = {} + if !item["status"].nil? && !item["status"].empty? + if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? + item["status"]["containerStatuses"].each do |containerStatus| + containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + containerName = containerStatus["name"] + # containeId format is :// + containerId = containerStatus["containerID"].split("//")[1] + containerInventoryRecord["InstanceID"] = containerId + # imagedId is of the format - repo@sha256:imageid + imageIdValue = containerStatus["imageID"] + if !imageIdValue.nil? && !imageIdValue.empty? + atLocation = imageIdValue.index("@") + if !atLocation.nil? + containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] + end + end + # image is of the format - repository/image:imagetag + imageValue = containerStatus["image"] + if !imageValue.nil? && !imageValue.empty? + # Find delimiters in the string of format repository/image:imagetag + slashLocation = imageValue.index("/") + colonLocation = imageValue.index(":") + if !colonLocation.nil? + if slashLocation.nil? + # image:imagetag + containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] + else + # repository/image:imagetag + containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] + containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] end + containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] + end + elsif !imageIdValue.nil? && !imageIdValue.empty? + # Getting repo information from imageIdValue when no tag in ImageId + if !atLocation.nil? + containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] + end + end + containerInventoryRecord["ExitCode"] = 0 + if !containerStatus["state"].nil? && !containerStatus["state"].empty? + containerState = containerStatus["state"] + if containerState.key?("running") + containerInventoryRecord["State"] = "Running" + containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] + elsif containerState.key?("terminated") + containerInventoryRecord["State"] = "Terminated" + containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] + containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] + containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] + elsif containerState.key?("waiting") + containerInventoryRecord["State"] = "Waiting" + end end + containerInfoMap = containersInfoMap[containerName] + containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] + containerInventoryRecord["Computer"] = containerInfoMap["Computer"] + containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] + containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] + containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] + containerInventoryRecord["Ports"] = containerInfoMap["Ports"] + containerInventoryRecord["Command"] = containerInfoMap["Command"] + containerInventoryRecords.push containerInventoryRecord + end end - rescue => error - @log.warn("KubeletUtils::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") + end end - return containerInventoryRecords + end end + rescue => error + @log.warn("KubeletUtils::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") + end + return containerInventoryRecords + end - def getContainersInfoMap(item, clusterCollectEnvironmentVar) - containersInfoMap = {} - begin - nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" - createdTime = item["metadata"]["creationTimestamp"] - if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? - if !item["spec"]["containers"].nil? && !item["spec"]["containers"].empty? - item["spec"]["containers"].each do |container| - containerInfoMap = {} - containerName = container["name"] - containerInfoMap["ElementName"] = containerName - containerInfoMap["Computer"] = nodeName - containerInfoMap["ContainerHostname"] = nodeName - containerInfoMap["CreatedTime"] = createdTime - if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 - containerInfoMap["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] - else - containerInfoMap["EnvironmentVar"] = parseContainerEnvironmentVarsJSON(containerName, container["env"]) - end - portsValue = container["ports"] - portsValueString = (portsValue.nil?) ? "" : portsValue.to_s - containerInfoMap["Ports"] = portsValueString - cmdValue = container["command"] - cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s - containerInfoMap["Command"] = cmdValueString + def getContainersInfoMap(item, clusterCollectEnvironmentVar) + containersInfoMap = {} + begin + nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" + createdTime = item["metadata"]["creationTimestamp"] + if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? + if !item["spec"]["containers"].nil? && !item["spec"]["containers"].empty? + item["spec"]["containers"].each do |container| + containerInfoMap = {} + containerName = container["name"] + containerInfoMap["ElementName"] = containerName + containerInfoMap["Computer"] = nodeName + containerInfoMap["ContainerHostname"] = nodeName + containerInfoMap["CreatedTime"] = createdTime + if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 + containerInfoMap["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] + else + containerInfoMap["EnvironmentVar"] = parseContainerEnvironmentVarsJSON(containerName, container["env"]) + end + portsValue = container["ports"] + portsValueString = (portsValue.nil?) ? "" : portsValue.to_s + containerInfoMap["Ports"] = portsValueString + cmdValue = container["command"] + cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s + containerInfoMap["Command"] = cmdValueString - containersInfoMap[containerName] = containerInfoMap - end - end - end - rescue => error - @log.warn("KubeletUtils::getContainersInfoMap : Get Container Info Maps failed: #{error}") + containersInfoMap[containerName] = containerInfoMap end - return containersInfoMap + end end + rescue => error + @log.warn("KubeletUtils::getContainersInfoMap : Get Container Info Maps failed: #{error}") + end + return containersInfoMap + end - def parseContainerEnvironmentVarsJSON(containerName, envVarsJSON) - envValueString = "" - begin - envVars = [] - if !envVarsJSON.nil? && !envVarsJSON.empty? - envVarsJSON.each do |envVar| - key = envVar["name"] - value = "" - if !envVar["value"].nil? - value = envVar["value"] - # elsif !envVar["valueFrom"].nil? - # valueFrom = envVar["valueFrom"] - # if valueFrom.key?("fieldRef") && !valueFrom["fieldRef"]["fieldPath"].nil? && !valueFrom["fieldRef"]["fieldPath"].empty? - # fieldPath = valueFrom["fieldRef"]["fieldPath"] - # fields = fieldPath.split('.') - # if fields.length() == 2 - # if !fields[1].nil? && !fields[1].empty? & fields[1].end_with?(']') - # indexFields = fields[1].split('[') - # hashMapValue = item[fields[0]][indexFields[0]] - # if !hashMapValue.nil? && !hashMapValue.empty? - # subField = indexFields[1].delete_suffix("]").delete("\\'") - # value = hashMapValue[subField] - # end - # else - # value = item[fields[0]][fields[1]] - # end - # end - # else - # value = envVar["valueFrom"].to_s - # end - end - envVars.push("#{key}=#{value}") - end - envValueString = envVars.to_s - # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE - # Check to see if the environment variable collection is disabled for this container. - if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) - envValueString = ["AZMON_COLLECT_ENV=FALSE"] - $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") - else - # Restricting the ENV string value to 200kb since the size of this string can go very high - if envValueString.length > 200000 - envValueStringTruncated = envValueString.slice(0..200000) - lastIndex = envValueStringTruncated.rindex("\", ") - if !lastIndex.nil? - envValueString = envValueStringTruncated.slice(0..lastIndex) + "]" - else - envValueString = envValueStringTruncated - end - end + def parseContainerEnvironmentVarsJSON(containerName, envVarsJSON) + envValueString = "" + begin + envVars = [] + if !envVarsJSON.nil? && !envVarsJSON.empty? + envVarsJSON.each do |envVar| + key = envVar["name"] + value = "" + if !envVar["value"].nil? + value = envVar["value"] + elsif !envVar["valueFrom"].nil? + valueFrom = envVar["valueFrom"] + if valueFrom.key?("fieldRef") && !valueFrom["fieldRef"]["fieldPath"].nil? && !valueFrom["fieldRef"]["fieldPath"].empty? + fieldPath = valueFrom["fieldRef"]["fieldPath"] + fields = fieldPath.split(".") + if fields.length() == 2 + if !fields[1].nil? && !fields[1].empty? & fields[1].end_with?("]") + indexFields = fields[1].split("[") + hashMapValue = item[fields[0]][indexFields[0]] + if !hashMapValue.nil? && !hashMapValue.empty? + subField = indexFields[1].delete_suffix("]").delete("\\'") + value = hashMapValue[subField] end + else + value = item[fields[0]][fields[1]] + end end - rescue => error - @log.warn("KubeletUtils::parseContainerEnvironmentVarsJSON : parsing of EnvVars JSON failed: #{error}") + else + value = envVar["valueFrom"].to_s + end + end + envVars.push("#{key}=#{value}") + end + envValueString = envVars.to_s + # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE + # Check to see if the environment variable collection is disabled for this container. + if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) + envValueString = ["AZMON_COLLECT_ENV=FALSE"] + $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") + else + # Restricting the ENV string value to 200kb since the size of this string can go very high + if envValueString.length > 200000 + envValueStringTruncated = envValueString.slice(0..200000) + lastIndex = envValueStringTruncated.rindex("\", ") + if !lastIndex.nil? + envValueString = envValueStringTruncated.slice(0..lastIndex) + "]" + else + envValueString = envValueStringTruncated + end end - return envValueString + end end + rescue => error + @log.warn("KubeletUtils::parseContainerEnvironmentVarsJSON : parsing of EnvVars JSON failed: #{error}") + end + return envValueString end -end \ No newline at end of file + end +end From 13bc75d336f30c5c477b31378e8bdf8f0a2f3763 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 21 Feb 2020 22:17:48 -0800 Subject: [PATCH 36/73] fix bug --- source/code/plugin/kubelet_utils.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 870729028..d467dcf27 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -121,7 +121,7 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 containerInfoMap["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] else - containerInfoMap["EnvironmentVar"] = parseContainerEnvironmentVarsJSON(containerName, container["env"]) + containerInfoMap["EnvironmentVar"] = parseContainerEnvironmentVarsJSON(item, containerName, container["env"]) end portsValue = container["ports"] portsValueString = (portsValue.nil?) ? "" : portsValue.to_s @@ -140,11 +140,11 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) return containersInfoMap end - def parseContainerEnvironmentVarsJSON(containerName, envVarsJSON) + def parseContainerEnvironmentVarsJSON(item, containerName, envVarsJSON) envValueString = "" begin envVars = [] - if !envVarsJSON.nil? && !envVarsJSON.empty? + if !item.nil? && !item.empty? && !envVarsJSON.nil? && !envVarsJSON.empty? envVarsJSON.each do |envVar| key = envVar["name"] value = "" @@ -156,7 +156,7 @@ def parseContainerEnvironmentVarsJSON(containerName, envVarsJSON) fieldPath = valueFrom["fieldRef"]["fieldPath"] fields = fieldPath.split(".") if fields.length() == 2 - if !fields[1].nil? && !fields[1].empty? & fields[1].end_with?("]") + if !fields[1].nil? && !fields[1].empty? && fields[1].end_with?("]") indexFields = fields[1].split("[") hashMapValue = item[fields[0]][indexFields[0]] if !hashMapValue.nil? && !hashMapValue.empty? From d5ff0771861fa87779248692eed7d01384380d55 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 22 Feb 2020 17:16:55 -0800 Subject: [PATCH 37/73] add resourcefieldref support in env --- source/code/plugin/kubelet_utils.rb | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index d467dcf27..f14d0a2c8 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -121,7 +121,7 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 containerInfoMap["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] else - containerInfoMap["EnvironmentVar"] = parseContainerEnvironmentVarsJSON(item, containerName, container["env"]) + containerInfoMap["EnvironmentVar"] = parseContainerEnvironmentVarsJSON(item, container) end portsValue = container["ports"] portsValueString = (portsValue.nil?) ? "" : portsValue.to_s @@ -140,11 +140,12 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) return containersInfoMap end - def parseContainerEnvironmentVarsJSON(item, containerName, envVarsJSON) + def parseContainerEnvironmentVarsJSON(pod, container) envValueString = "" begin envVars = [] - if !item.nil? && !item.empty? && !envVarsJSON.nil? && !envVarsJSON.empty? + envVarsJSON = container["env"] + if !pod.nil? && !pod.empty? && !envVarsJSON.nil? && !envVarsJSON.empty? envVarsJSON.each do |envVar| key = envVar["name"] value = "" @@ -152,21 +153,30 @@ def parseContainerEnvironmentVarsJSON(item, containerName, envVarsJSON) value = envVar["value"] elsif !envVar["valueFrom"].nil? valueFrom = envVar["valueFrom"] + # https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-pod-fields-as-values-for-environment-variables if valueFrom.key?("fieldRef") && !valueFrom["fieldRef"]["fieldPath"].nil? && !valueFrom["fieldRef"]["fieldPath"].empty? fieldPath = valueFrom["fieldRef"]["fieldPath"] fields = fieldPath.split(".") if fields.length() == 2 if !fields[1].nil? && !fields[1].empty? && fields[1].end_with?("]") indexFields = fields[1].split("[") - hashMapValue = item[fields[0]][indexFields[0]] + hashMapValue = pod[fields[0]][indexFields[0]] if !hashMapValue.nil? && !hashMapValue.empty? subField = indexFields[1].delete_suffix("]").delete("\\'") value = hashMapValue[subField] end else - value = item[fields[0]][fields[1]] + value = pod[fields[0]][fields[1]] end end + # https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-container-fields-as-values-for-environment-variables + elsif valueFrom.key?("resourceFieldRef") && !valueFrom["resourceFieldRef"]["resource"].nil? && !valueFrom["resourceFieldRef"]["resource"].empty? + resource = valueFrom["resourceFieldRef"]["resource"] + resourceFileds = resource.split(".") + containerResources = container["resources"] + if !containerResources.nil? && !containerResources.empty? && resourceFileds.length() == 2 + value = containerResources[resourceFileds[0]][resourceFileds[1]] + end else value = envVar["valueFrom"].to_s end @@ -174,6 +184,7 @@ def parseContainerEnvironmentVarsJSON(item, containerName, envVarsJSON) envVars.push("#{key}=#{value}") end envValueString = envVars.to_s + containerName = container["name"] # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE # Check to see if the environment variable collection is disabled for this container. if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) @@ -193,7 +204,7 @@ def parseContainerEnvironmentVarsJSON(item, containerName, envVarsJSON) end end rescue => error - @log.warn("KubeletUtils::parseContainerEnvironmentVarsJSON : parsing of EnvVars JSON failed: #{error}") + @log.warn("KubeletUtils::parseContainerEnvironmentVarsJSON : parsing of EnvVars failed: #{error}") end return envValueString end From fca1ce5ca1b73d06d1c7873cd9fe3a97376b888a Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 22 Feb 2020 22:34:29 -0800 Subject: [PATCH 38/73] include init containers as well --- source/code/plugin/kubelet_utils.rb | 132 +++++++++++++++------------- 1 file changed, 72 insertions(+), 60 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index f14d0a2c8..6cfb1e27e 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -28,71 +28,76 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? podList["items"].each do |item| containersInfoMap = getContainersInfoMap(item, clusterCollectEnvironmentVar) + podContainersStatuses = [] + if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? + podContainersStatuses = item["status"]["containerStatuses"] + end + if !item["status"]["initContainerStatuses"].nil? && !item["status"]["initContainerStatuses"].empty? + podContainersStatuses = podContainersStatuses + item["status"]["initContainerStatuses"] + end containerInventoryRecord = {} - if !item["status"].nil? && !item["status"].empty? - if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? - item["status"]["containerStatuses"].each do |containerStatus| - containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated - containerName = containerStatus["name"] - # containeId format is :// - containerId = containerStatus["containerID"].split("//")[1] - containerInventoryRecord["InstanceID"] = containerId - # imagedId is of the format - repo@sha256:imageid - imageIdValue = containerStatus["imageID"] - if !imageIdValue.nil? && !imageIdValue.empty? - atLocation = imageIdValue.index("@") - if !atLocation.nil? - containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] - end + if !PodContainersStatuses.empty? + PodContainersStatuses.each do |containerStatus| + containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + containerName = containerStatus["name"] + # containeId format is :// + containerId = containerStatus["containerID"].split("//")[1] + containerInventoryRecord["InstanceID"] = containerId + # imagedId is of the format - repo@sha256:imageid + imageIdValue = containerStatus["imageID"] + if !imageIdValue.nil? && !imageIdValue.empty? + atLocation = imageIdValue.index("@") + if !atLocation.nil? + containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] end - # image is of the format - repository/image:imagetag - imageValue = containerStatus["image"] - if !imageValue.nil? && !imageValue.empty? - # Find delimiters in the string of format repository/image:imagetag - slashLocation = imageValue.index("/") - colonLocation = imageValue.index(":") - if !colonLocation.nil? - if slashLocation.nil? - # image:imagetag - containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] - else - # repository/image:imagetag - containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] - containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] - end - containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] - end - elsif !imageIdValue.nil? && !imageIdValue.empty? - # Getting repo information from imageIdValue when no tag in ImageId - if !atLocation.nil? - containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] + end + # image is of the format - repository/image:imagetag + imageValue = containerStatus["image"] + if !imageValue.nil? && !imageValue.empty? + # Find delimiters in the string of format repository/image:imagetag + slashLocation = imageValue.index("/") + colonLocation = imageValue.index(":") + if !colonLocation.nil? + if slashLocation.nil? + # image:imagetag + containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] + else + # repository/image:imagetag + containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] + containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] end + containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] end - containerInventoryRecord["ExitCode"] = 0 - if !containerStatus["state"].nil? && !containerStatus["state"].empty? - containerState = containerStatus["state"] - if containerState.key?("running") - containerInventoryRecord["State"] = "Running" - containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] - elsif containerState.key?("terminated") - containerInventoryRecord["State"] = "Terminated" - containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] - containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] - containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] - elsif containerState.key?("waiting") - containerInventoryRecord["State"] = "Waiting" - end + elsif !imageIdValue.nil? && !imageIdValue.empty? + # Getting repo information from imageIdValue when no tag in ImageId + if !atLocation.nil? + containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] + end + end + containerInventoryRecord["ExitCode"] = 0 + if !containerStatus["state"].nil? && !containerStatus["state"].empty? + containerState = containerStatus["state"] + if containerState.key?("running") + containerInventoryRecord["State"] = "Running" + containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] + elsif containerState.key?("terminated") + containerInventoryRecord["State"] = "Terminated" + containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] + containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] + containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] + elsif containerState.key?("waiting") + containerInventoryRecord["State"] = "Waiting" end - containerInfoMap = containersInfoMap[containerName] - containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] - containerInventoryRecord["Computer"] = containerInfoMap["Computer"] - containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] - containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] - containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] - containerInventoryRecord["Ports"] = containerInfoMap["Ports"] - containerInventoryRecord["Command"] = containerInfoMap["Command"] - containerInventoryRecords.push containerInventoryRecord end + containerInfoMap = containersInfoMap[containerName] + containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] + containerInventoryRecord["Computer"] = containerInfoMap["Computer"] + containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] + containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] + containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] + containerInventoryRecord["Ports"] = containerInfoMap["Ports"] + containerInventoryRecord["Command"] = containerInfoMap["Command"] + containerInventoryRecords.push containerInventoryRecord end end end @@ -110,8 +115,15 @@ def getContainersInfoMap(item, clusterCollectEnvironmentVar) nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" createdTime = item["metadata"]["creationTimestamp"] if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? + podContainers = [] if !item["spec"]["containers"].nil? && !item["spec"]["containers"].empty? - item["spec"]["containers"].each do |container| + podContainers = item["spec"]["containers"] + end + if !item["spec"]["initContainers"].nil? && !item["spec"]["initContainers"].empty? + podContainers = podContainers + item["spec"]["initContainers"] + end + if !podContainers.empty? + podContainers.each do |container| containerInfoMap = {} containerName = container["name"] containerInfoMap["ElementName"] = containerName From 765d3528234a55ef7d4d0aa84ee54f91f8fa0bbb Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 22 Feb 2020 23:20:23 -0800 Subject: [PATCH 39/73] fix bug --- source/code/plugin/kubelet_utils.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 6cfb1e27e..60b648f6b 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -36,8 +36,8 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) podContainersStatuses = podContainersStatuses + item["status"]["initContainerStatuses"] end containerInventoryRecord = {} - if !PodContainersStatuses.empty? - PodContainersStatuses.each do |containerStatus| + if !podContainersStatuses.empty? + podContainersStatuses.each do |containerStatus| containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated containerName = containerStatus["name"] # containeId format is :// From 73b694a0f9f529255bf3f71ef568853a31cc8ea8 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 23 Feb 2020 17:09:19 -0800 Subject: [PATCH 40/73] fix typo --- source/code/plugin/kubelet_utils.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 60b648f6b..611efb643 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -184,10 +184,10 @@ def parseContainerEnvironmentVarsJSON(pod, container) # https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-container-fields-as-values-for-environment-variables elsif valueFrom.key?("resourceFieldRef") && !valueFrom["resourceFieldRef"]["resource"].nil? && !valueFrom["resourceFieldRef"]["resource"].empty? resource = valueFrom["resourceFieldRef"]["resource"] - resourceFileds = resource.split(".") + resourceFields = resource.split(".") containerResources = container["resources"] - if !containerResources.nil? && !containerResources.empty? && resourceFileds.length() == 2 - value = containerResources[resourceFileds[0]][resourceFileds[1]] + if !containerResources.nil? && !containerResources.empty? && resourceFields.length() == 2 + value = containerResources[resourceFields[0]][resourceFields[1]] end else value = envVar["valueFrom"].to_s From 7f00a87ebfbb6cd1c0a9b4c82e53bef90982face Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 23 Feb 2020 17:58:36 -0800 Subject: [PATCH 41/73] use chomp instead of delete_suffix --- source/code/plugin/kubelet_utils.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 611efb643..4f2345eb9 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -174,7 +174,7 @@ def parseContainerEnvironmentVarsJSON(pod, container) indexFields = fields[1].split("[") hashMapValue = pod[fields[0]][indexFields[0]] if !hashMapValue.nil? && !hashMapValue.empty? - subField = indexFields[1].delete_suffix("]").delete("\\'") + subField = indexFields[1].chomp("]").delete("\\'") value = hashMapValue[subField] end else From c95a4571c2f43ab70d73df9501df6a26f9dea1b3 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 24 Feb 2020 13:23:11 -0800 Subject: [PATCH 42/73] add secretkeyref support --- source/code/plugin/kubelet_utils.rb | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 4f2345eb9..f43103a85 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -189,6 +189,14 @@ def parseContainerEnvironmentVarsJSON(pod, container) if !containerResources.nil? && !containerResources.empty? && resourceFields.length() == 2 value = containerResources[resourceFields[0]][resourceFields[1]] end + # https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables + elsif valueFrom.key?("secretKeyRef") + secretName = valueFrom["secretKeyRef"]["name"] + secretKey = valueFrom["secretKeyRef"]["key"] + # flatten value so that Ux can show that + if !secretName.nil && !secretName.empty && !secretKey.nil? && !secretKey.empty? + value = "secretKeyRef_#{secretName}_#{secretKey}" + end else value = envVar["valueFrom"].to_s end From 03796ae17669b1ad90ae96ea3cf118dcd551fa4e Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 24 Feb 2020 13:23:56 -0800 Subject: [PATCH 43/73] add secretkeyref support --- source/code/plugin/kubelet_utils.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index f43103a85..15ce7f7ec 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -193,7 +193,7 @@ def parseContainerEnvironmentVarsJSON(pod, container) elsif valueFrom.key?("secretKeyRef") secretName = valueFrom["secretKeyRef"]["name"] secretKey = valueFrom["secretKeyRef"]["key"] - # flatten value so that Ux can show that + # This is just secret not the plaintext. Flatten value so that Ux can show that if !secretName.nil && !secretName.empty && !secretKey.nil? && !secretKey.empty? value = "secretKeyRef_#{secretName}_#{secretKey}" end From aa912b05ce29fbef6c462c3b5e10c9abd808b1cc Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 24 Feb 2020 15:17:58 -0800 Subject: [PATCH 44/73] fix bug --- source/code/plugin/kubelet_utils.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 15ce7f7ec..47ddd2b83 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -193,8 +193,8 @@ def parseContainerEnvironmentVarsJSON(pod, container) elsif valueFrom.key?("secretKeyRef") secretName = valueFrom["secretKeyRef"]["name"] secretKey = valueFrom["secretKeyRef"]["key"] - # This is just secret not the plaintext. Flatten value so that Ux can show that - if !secretName.nil && !secretName.empty && !secretKey.nil? && !secretKey.empty? + # This is still secret not the plaintext. Flatten value so that CI Ux can show that + if !secretName.nil? && !secretName.empty? && !secretKey.nil? && !secretKey.empty? value = "secretKeyRef_#{secretName}_#{secretKey}" end else From 213adacc9b20f96fe28020841682ad956af29fd3 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 25 Feb 2020 11:16:43 -0800 Subject: [PATCH 45/73] handle dockerversion for non-docker runtimes --- source/code/plugin/in_kube_nodes.rb | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/source/code/plugin/in_kube_nodes.rb b/source/code/plugin/in_kube_nodes.rb index 6a099d626..351aee956 100644 --- a/source/code/plugin/in_kube_nodes.rb +++ b/source/code/plugin/in_kube_nodes.rb @@ -160,9 +160,12 @@ def parse_and_emit_records(nodeInventory, batchTime = Time.utc.iso8601) record["KubeletVersion"] = nodeInfo["kubeletVersion"] record["KubeProxyVersion"] = nodeInfo["kubeProxyVersion"] containerNodeInventoryRecord["OperatingSystem"] = nodeInfo["osImage"] - containerRuntimeVersion = nodeInfo["containerRuntimeVersion"] - if containerRuntimeVersion.downcase.start_with?("docker://") + containerRuntimeVersion = nodeInfo["containerRuntimeVersion"] + if containerRuntimeVersion.downcase.start_with?("docker://") containerNodeInventoryRecord["DockerVersion"] = containerRuntimeVersion.split("//")[1] + else + # using containerRuntimeVersion as DockerVersion as is for non docker runtimes + containerNodeInventoryRecord["DockerVersion"] = containerRuntimeVersion end # ContainerNodeInventory data for docker version and operating system. containerNodeInventoryWrapper = { @@ -187,9 +190,11 @@ def parse_and_emit_records(nodeInventory, batchTime = Time.utc.iso8601) properties["KubeletVersion"] = record["KubeletVersion"] properties["OperatingSystem"] = nodeInfo["operatingSystem"] properties["ContainerRuntimeVersion"] = containerRuntimeVersion - if containerRuntimeVersion.downcase.start_with?("docker://") + if containerRuntimeVersion.downcase.start_with?("docker://") properties["DockerVersion"] = containerRuntimeVersion.split("//")[1] - end + else + properties["DockerVersion"] = containerRuntimeVersion + end properties["KubernetesProviderID"] = record["KubernetesProviderID"] properties["KernelVersion"] = nodeInfo["kernelVersion"] properties["OSImage"] = nodeInfo["osImage"] From 94b589f37ee52767c838d12e6dd74f715dc67c1d Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 25 Feb 2020 17:49:22 -0800 Subject: [PATCH 46/73] fix comment --- source/code/go/src/plugins/oms.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 4e7bd6917..0713416c6 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -99,7 +99,7 @@ var ( enrichContainerLogs bool // container runtime engine configured on the kubelet (true indicates docker else K8s CRI compatiable runtime) isDockerContainerRuntimeEngine bool - // container runtime engine configured on the kubelet (true indicates docker else K8s CRI compatiable runtime) + // container runtime engine configured on the kubelet containerRuntime string ) From 10aac9e9ec50aebb5c8de731b3f16ccd9116deea Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 3 Mar 2020 12:48:46 -0800 Subject: [PATCH 47/73] handle kubelet metrics to handle runtime and k8s versions --- installer/conf/telegraf.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/installer/conf/telegraf.conf b/installer/conf/telegraf.conf index f9dc3fb6a..6207aa815 100644 --- a/installer/conf/telegraf.conf +++ b/installer/conf/telegraf.conf @@ -386,7 +386,7 @@ # report_active = true # fieldpass = ["usage_active","cluster","node","host","device"] # taginclude = ["cluster","cpu","node"] - + # Read metrics about disk usage by mount point @@ -395,7 +395,7 @@ ## By default stats will be gathered for all mount points. ## Set mount_points will restrict the stats to only the specified mount points. # mount_points = ["/"] - + ## Ignore mount points by filesystem type. ignore_fs = ["tmpfs", "devtmpfs", "devfs", "overlay", "aufs", "squashfs"] fieldpass = ["free", "used", "used_percent"] @@ -532,7 +532,7 @@ name_prefix="container.azm.ms/" ## An array of urls to scrape metrics from. urls = ["$CADVISOR_METRICS_URL"] - fieldpass = ["kubelet_docker_operations", "kubelet_docker_operations_errors"] + fieldpass = ["$KUBELET_RUNTIME_OPERATIONS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC"] metric_version = 2 url_tag = "scrapeUrl" @@ -578,7 +578,7 @@ urls = $AZMON_DS_PROM_URLS fieldpass = $AZMON_DS_PROM_FIELDPASS - + fielddrop = $AZMON_DS_PROM_FIELDDROP metric_version = 2 From 8a114d1728bd684404ad0bcd4d432525f4661f7d Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 4 Mar 2020 14:23:28 -0800 Subject: [PATCH 48/73] remove _total metrics until we support 1.18 --- installer/conf/telegraf.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/installer/conf/telegraf.conf b/installer/conf/telegraf.conf index 6207aa815..b7c3d44b3 100644 --- a/installer/conf/telegraf.conf +++ b/installer/conf/telegraf.conf @@ -532,7 +532,8 @@ name_prefix="container.azm.ms/" ## An array of urls to scrape metrics from. urls = ["$CADVISOR_METRICS_URL"] - fieldpass = ["$KUBELET_RUNTIME_OPERATIONS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC"] + ## Include "$KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC" when we add for support for 1.18 + fieldpass = ["$KUBELET_RUNTIME_OPERATIONS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC"] metric_version = 2 url_tag = "scrapeUrl" From 06a51cf3c1b061f96c614a3bc2fc8e6c4792ae69 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 6 Mar 2020 07:58:33 -0800 Subject: [PATCH 49/73] fetch envvars via proc environ --- source/code/plugin/in_containerinventory.rb | 280 ++++++++++++++++---- source/code/plugin/kubelet_utils.rb | 233 +--------------- 2 files changed, 240 insertions(+), 273 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index b5b8cee5f..e2bf7625e 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -13,13 +13,13 @@ class Container_Inventory_Input < Input def initialize super - require 'yajl/json_gem' + require "yajl/json_gem" require "time" require_relative "DockerApiClient" require_relative "ContainerInventoryState" require_relative "ApplicationInsightsUtility" - require_relative "omslog" - require_relative "kubelet_utils" + require_relative "omslog" + require_relative "CAdvisorMetricsAPIClient" end config_param :run_interval, :time, :default => 60 @@ -35,7 +35,9 @@ def start @condition = ConditionVariable.new @mutex = Mutex.new @thread = Thread.new(&method(:run_periodic)) - @@telemetryTimeTracker = DateTime.now.to_time.to_i + @@telemetryTimeTracker = DateTime.now.to_time.to_i + # cache the container and cgroup parent process + @containerCGroupCache = Hash.new end end @@ -189,9 +191,182 @@ def inspectContainer(id, nameMap, clusterCollectEnvironmentVar) $log.warn("Exception in inspectContainer: #{errorStr} for container: #{id}") end return containerInstance - end + end + + def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + containerInventoryRecords = Array.new + begin + response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) + if !response.nil? && !response.body.nil? + podList = JSON.parse(response.body) + if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? + podList["items"].each do |item| + containersInfoMap = getContainersInfoMap(item) + containerInventoryRecord = {} + if !item["status"].nil? && !item["status"].empty? + if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? + item["status"]["containerStatuses"].each do |containerStatus| + containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + containerName = containerStatus["name"] + containerId = containerStatus["containerID"].split("//")[1] + containerInventoryRecord["InstanceID"] = containerId + # imagedId is of the format - repo@sha256:imageid + imageIdValue = containerStatus["imageID"] + if !imageIdValue.nil? && !imageIdValue.empty? + atLocation = imageIdValue.index("@") + if !atLocation.nil? + containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] + end + end + # image is of the format - repository/image:imagetag + imageValue = containerStatus["image"] + if !imageValue.nil? && !imageValue.empty? + # Find delimiters in the string of format repository/image:imagetag + slashLocation = imageValue.index("/") + colonLocation = imageValue.index(":") + if !colonLocation.nil? + if slashLocation.nil? + # image:imagetag + containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] + else + # repository/image:imagetag + containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] + containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] + end + containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] + end + elsif !imageIdValue.nil? && !imageIdValue.empty? + # Getting repo information from imageIdValue when no tag in ImageId + if !atLocation.nil? + containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] + end + end + containerInventoryRecord["ExitCode"] = 0 + if !containerStatus["state"].nil? && !containerStatus["state"].empty? + containerState = containerStatus["state"] + if containerState.key?("running") + containerInventoryRecord["State"] = "Running" + containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] + elsif containerState.key?("terminated") + containerInventoryRecord["State"] = "Terminated" + containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] + containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] + containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] + elsif containerState.key?("waiting") + containerInventoryRecord["State"] = "Waiting" + end + end + containerInfoMap = containersInfoMap[containerName] + containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] + containerInventoryRecord["Computer"] = containerInfoMap["Computer"] + containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] + containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] + containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] + containerInventoryRecord["Ports"] = containerInfoMap["Ports"] + containerInventoryRecord["Command"] = containerInfoMap["Command"] + if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 + containerInventoryRecord["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] + else + containerInventoryRecord["EnvironmentVar"] = obtainContainerEnvironmentVars(containerId) + end + containerInventoryRecords.push containerInventoryRecord + end + end + end + end + end + end + rescue => error + @log.warn("in_container_inventory::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") + end + return containerInventoryRecords + end + + def getContainersInfoMap(item) + containersInfoMap = {} + begin + nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" + createdTime = item["metadata"]["creationTimestamp"] + if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? + if !item["spec"]["containers"].nil? && !item["spec"]["containers"].empty? + item["spec"]["containers"].each do |container| + containerInfoMap = {} + containerName = container["name"] + containerInfoMap["ElementName"] = containerName + containerInfoMap["Computer"] = nodeName + containerInfoMap["ContainerHostname"] = nodeName + containerInfoMap["CreatedTime"] = createdTime + portsValue = container["ports"] + portsValueString = (portsValue.nil?) ? "" : portsValue.to_s + containerInfoMap["Ports"] = portsValueString + cmdValue = container["command"] + cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s + containerInfoMap["Command"] = cmdValueString + + containersInfoMap[containerName] = containerInfoMap + end + end + end + rescue => error + @log.warn("in_container_inventory::getContainersInfoMap : Get Container Info Maps failed: #{error}") + end + return containersInfoMap + end + + def obtainContainerEnvironmentVars(containerId) + $log.info("in_container_inventory::obtainContainerEnvironmentVars @ #{Time.now.utc.iso8601}") + envValueString = "" + begin + unless @containerCGroupCache.has_key?(containerId) + $log.info("in_container_inventory::fetching cGroup parent pid @ #{Time.now.utc.iso8601}") + Dir["/hostfs/proc/*/cgroup"].each do |filename| + if File.file?(filename) && File.foreach(filename).grep(/#{containerId}/).any? + # file full path is /hostfs/proc//cgroup + $log.info("in_container_inventory::fetching cGroup parent filename @ #{filename}") + cGroupPid = filename.split("/")[3] + if @containerCGroupCache.has_key?(containerId) + tempCGroupPid = containerCgroupCache[containerId] + if tempCGroupPid > cGroupPid + @containerCgroupCache[containerId] = cGroupPid + end + else + @containerCGroupCache[containerId] = cGroupPid + end + end + end + end + cGroupPid = @containerCGroupCache[containerId] + if !cGroupPid.nil? + environFilePath = "/hostfs/proc/#{cGroupPid}/environ" + if File.exist?(environFilePath) + # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE + # Check to see if the environment variable collection is disabled for this container. + if File.foreach(environFilePath).grep(/AZMON_COLLECT_ENV=FALSE/i).any? + envValueString = ["AZMON_COLLECT_ENV=FALSE"] + $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") + else + fileSize = File.size(environFilePath) + $log.info("in_container_inventory::environment vars filename @ #{filename} filesize @ #{fileSize}") + # Restricting the ENV string value to 200kb since the size of this string can go very high + envVars = File.read(environFilePath, 200000).split(" ") + envValueString = envVars.to_s + if fileSize > 200000 + lastIndex = envValueString.rindex("\", ") + if !lastIndex.nil? + envValueStringTruncated = envValueString.slice(0..lastIndex) + "]" + envValueString = envValueStringTruncated + end + end + end + end + end + rescue => error + @log.warn("in_container_inventory::obtainContainerEnvironmentVars : obtain Container Environment vars failed: #{error} for containerId: #{containerId}") + end + return envValueString + end - def enumerate + def enumerate currentTime = Time.now emitTime = currentTime.to_f batchTime = currentTime.utc.iso8601 @@ -204,59 +379,60 @@ def enumerate $log.info("in_container_inventory::enumerate : container runtime : #{containerRuntimeEnv}") clusterCollectEnvironmentVar = ENV["AZMON_CLUSTER_COLLECT_ENV_VAR"] if !containerRuntimeEnv.nil? && !containerRuntimeEnv.empty? && containerRuntimeEnv.casecmp("docker") == 0 - $log.info("in_container_inventory::enumerate : using docker apis since container runtime is docker") - hostName = DockerApiClient.getDockerHostName - containerIds = DockerApiClient.listContainers - if !containerIds.nil? && !containerIds.empty? - nameMap = DockerApiClient.getImageIdMap - if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 - $log.warn("Environment Variable collection disabled for cluster") - end - containerIds.each do |containerId| - inspectedContainer = {} - inspectedContainer = inspectContainer(containerId, nameMap, clusterCollectEnvironmentVar) - inspectedContainer["Computer"] = hostName - inspectedContainer["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated - containerInventory.push inspectedContainer - ContainerInventoryState.writeContainerState(inspectedContainer) - end - # Update the state for deleted containers - deletedContainers = ContainerInventoryState.getDeletedContainers(containerIds) - if !deletedContainers.nil? && !deletedContainers.empty? - deletedContainers.each do |deletedContainer| - container = ContainerInventoryState.readContainerState(deletedContainer) - if !container.nil? - container.each { |k, v| container[k] = v } - container["State"] = "Deleted" - containerInventory.push container - end - end - end + $log.info("in_container_inventory::enumerate : using docker apis since container runtime is docker") + hostName = DockerApiClient.getDockerHostName + containerIds = DockerApiClient.listContainers + if !containerIds.nil? && !containerIds.empty? + nameMap = DockerApiClient.getImageIdMap + if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 + $log.warn("Environment Variable collection disabled for cluster") end - else - $log.info("in_container_inventory::enumerate : using cadvisor apis since non docker container runtime : #{containerRuntimeEnv}") - containerInventoryRecords = KubeletUtils.getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) - containerIds = Array.new - containerInventoryRecords.each do |containerRecord| - ContainerInventoryState.writeContainerState(containerRecord) - if hostName.empty? && !containerRecord["Computer"].empty? - hostName = containerRecord["Computer"] - end - containerIds.push containerRecord["InstanceID"] - containerInventory.push containerRecord + containerIds.each do |containerId| + inspectedContainer = {} + inspectedContainer = inspectContainer(containerId, nameMap, clusterCollectEnvironmentVar) + inspectedContainer["Computer"] = hostName + inspectedContainer["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + containerInventory.push inspectedContainer + ContainerInventoryState.writeContainerState(inspectedContainer) end # Update the state for deleted containers deletedContainers = ContainerInventoryState.getDeletedContainers(containerIds) if !deletedContainers.nil? && !deletedContainers.empty? - deletedContainers.each do |deletedContainer| - container = ContainerInventoryState.readContainerState(deletedContainer) - if !container.nil? - container.each { |k, v| container[k] = v } - container["State"] = "Deleted" - containerInventory.push container - end + deletedContainers.each do |deletedContainer| + container = ContainerInventoryState.readContainerState(deletedContainer) + if !container.nil? + container.each { |k, v| container[k] = v } + container["State"] = "Deleted" + containerInventory.push container end + end end + end + else + $log.info("in_container_inventory::enumerate : using cadvisor apis since non docker container runtime : #{containerRuntimeEnv}") + containerInventoryRecords = getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) + containerIds = Array.new + containerInventoryRecords.each do |containerRecord| + ContainerInventoryState.writeContainerState(containerRecord) + if hostName.empty? && !containerRecord["Computer"].empty? + hostName = containerRecord["Computer"] + end + containerIds.push containerRecord["InstanceID"] + containerInventory.push containerRecord + end + # Update the state for deleted containers + deletedContainers = ContainerInventoryState.getDeletedContainers(containerIds) + if !deletedContainers.nil? && !deletedContainers.empty? + deletedContainers.each do |deletedContainer| + container = ContainerInventoryState.readContainerState(deletedContainer) + if !container.nil? + container.each { |k, v| container[k] = v } + container["State"] = "Deleted" + @containerCGroupCache.delete(container["InstanceID"]) + containerInventory.push container + end + end + end end containerInventory.each do |record| diff --git a/source/code/plugin/kubelet_utils.rb b/source/code/plugin/kubelet_utils.rb index 47ddd2b83..fecb339b9 100644 --- a/source/code/plugin/kubelet_utils.rb +++ b/source/code/plugin/kubelet_utils.rb @@ -2,231 +2,22 @@ #!/usr/local/bin/ruby # frozen_string_literal: true -require_relative "CAdvisorMetricsAPIClient" +require_relative 'CAdvisorMetricsAPIClient' class KubeletUtils - class << self - def get_node_capacity - cpu_capacity = 1.0 - memory_capacity = 1.0 + class << self + def get_node_capacity - response = CAdvisorMetricsAPIClient.getNodeCapacityFromCAdvisor(winNode: nil) - if !response.nil? && !response.body.nil? - cpu_capacity = JSON.parse(response.body)["num_cores"].nil? ? 1.0 : (JSON.parse(response.body)["num_cores"] * 1000.0) - memory_capacity = JSON.parse(response.body)["memory_capacity"].nil? ? 1.0 : JSON.parse(response.body)["memory_capacity"].to_f - $log.info "CPU = #{cpu_capacity}mc Memory = #{memory_capacity / 1024 / 1024}MB" - return [cpu_capacity, memory_capacity] - end - end - - def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) - containerInventoryRecords = Array.new - begin - response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) - if !response.nil? && !response.body.nil? - podList = JSON.parse(response.body) - if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? - podList["items"].each do |item| - containersInfoMap = getContainersInfoMap(item, clusterCollectEnvironmentVar) - podContainersStatuses = [] - if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? - podContainersStatuses = item["status"]["containerStatuses"] - end - if !item["status"]["initContainerStatuses"].nil? && !item["status"]["initContainerStatuses"].empty? - podContainersStatuses = podContainersStatuses + item["status"]["initContainerStatuses"] - end - containerInventoryRecord = {} - if !podContainersStatuses.empty? - podContainersStatuses.each do |containerStatus| - containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated - containerName = containerStatus["name"] - # containeId format is :// - containerId = containerStatus["containerID"].split("//")[1] - containerInventoryRecord["InstanceID"] = containerId - # imagedId is of the format - repo@sha256:imageid - imageIdValue = containerStatus["imageID"] - if !imageIdValue.nil? && !imageIdValue.empty? - atLocation = imageIdValue.index("@") - if !atLocation.nil? - containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] - end - end - # image is of the format - repository/image:imagetag - imageValue = containerStatus["image"] - if !imageValue.nil? && !imageValue.empty? - # Find delimiters in the string of format repository/image:imagetag - slashLocation = imageValue.index("/") - colonLocation = imageValue.index(":") - if !colonLocation.nil? - if slashLocation.nil? - # image:imagetag - containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] - else - # repository/image:imagetag - containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] - containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] - end - containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] - end - elsif !imageIdValue.nil? && !imageIdValue.empty? - # Getting repo information from imageIdValue when no tag in ImageId - if !atLocation.nil? - containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] - end - end - containerInventoryRecord["ExitCode"] = 0 - if !containerStatus["state"].nil? && !containerStatus["state"].empty? - containerState = containerStatus["state"] - if containerState.key?("running") - containerInventoryRecord["State"] = "Running" - containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] - elsif containerState.key?("terminated") - containerInventoryRecord["State"] = "Terminated" - containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] - containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] - containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] - elsif containerState.key?("waiting") - containerInventoryRecord["State"] = "Waiting" - end - end - containerInfoMap = containersInfoMap[containerName] - containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] - containerInventoryRecord["Computer"] = containerInfoMap["Computer"] - containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] - containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] - containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] - containerInventoryRecord["Ports"] = containerInfoMap["Ports"] - containerInventoryRecord["Command"] = containerInfoMap["Command"] - containerInventoryRecords.push containerInventoryRecord - end - end - end - end - end - rescue => error - @log.warn("KubeletUtils::getContainerInventoryRecords : Get Container Inventory Records failed: #{error}") - end - return containerInventoryRecords - end - - def getContainersInfoMap(item, clusterCollectEnvironmentVar) - containersInfoMap = {} - begin - nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" - createdTime = item["metadata"]["creationTimestamp"] - if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? - podContainers = [] - if !item["spec"]["containers"].nil? && !item["spec"]["containers"].empty? - podContainers = item["spec"]["containers"] - end - if !item["spec"]["initContainers"].nil? && !item["spec"]["initContainers"].empty? - podContainers = podContainers + item["spec"]["initContainers"] - end - if !podContainers.empty? - podContainers.each do |container| - containerInfoMap = {} - containerName = container["name"] - containerInfoMap["ElementName"] = containerName - containerInfoMap["Computer"] = nodeName - containerInfoMap["ContainerHostname"] = nodeName - containerInfoMap["CreatedTime"] = createdTime - if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 - containerInfoMap["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] - else - containerInfoMap["EnvironmentVar"] = parseContainerEnvironmentVarsJSON(item, container) - end - portsValue = container["ports"] - portsValueString = (portsValue.nil?) ? "" : portsValue.to_s - containerInfoMap["Ports"] = portsValueString - cmdValue = container["command"] - cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s - containerInfoMap["Command"] = cmdValueString + cpu_capacity = 1.0 + memory_capacity = 1.0 - containersInfoMap[containerName] = containerInfoMap - end - end - end - rescue => error - @log.warn("KubeletUtils::getContainersInfoMap : Get Container Info Maps failed: #{error}") - end - return containersInfoMap - end - - def parseContainerEnvironmentVarsJSON(pod, container) - envValueString = "" - begin - envVars = [] - envVarsJSON = container["env"] - if !pod.nil? && !pod.empty? && !envVarsJSON.nil? && !envVarsJSON.empty? - envVarsJSON.each do |envVar| - key = envVar["name"] - value = "" - if !envVar["value"].nil? - value = envVar["value"] - elsif !envVar["valueFrom"].nil? - valueFrom = envVar["valueFrom"] - # https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-pod-fields-as-values-for-environment-variables - if valueFrom.key?("fieldRef") && !valueFrom["fieldRef"]["fieldPath"].nil? && !valueFrom["fieldRef"]["fieldPath"].empty? - fieldPath = valueFrom["fieldRef"]["fieldPath"] - fields = fieldPath.split(".") - if fields.length() == 2 - if !fields[1].nil? && !fields[1].empty? && fields[1].end_with?("]") - indexFields = fields[1].split("[") - hashMapValue = pod[fields[0]][indexFields[0]] - if !hashMapValue.nil? && !hashMapValue.empty? - subField = indexFields[1].chomp("]").delete("\\'") - value = hashMapValue[subField] - end - else - value = pod[fields[0]][fields[1]] - end - end - # https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-container-fields-as-values-for-environment-variables - elsif valueFrom.key?("resourceFieldRef") && !valueFrom["resourceFieldRef"]["resource"].nil? && !valueFrom["resourceFieldRef"]["resource"].empty? - resource = valueFrom["resourceFieldRef"]["resource"] - resourceFields = resource.split(".") - containerResources = container["resources"] - if !containerResources.nil? && !containerResources.empty? && resourceFields.length() == 2 - value = containerResources[resourceFields[0]][resourceFields[1]] - end - # https://kubernetes.io/docs/concepts/configuration/secret/#using-secrets-as-environment-variables - elsif valueFrom.key?("secretKeyRef") - secretName = valueFrom["secretKeyRef"]["name"] - secretKey = valueFrom["secretKeyRef"]["key"] - # This is still secret not the plaintext. Flatten value so that CI Ux can show that - if !secretName.nil? && !secretName.empty? && !secretKey.nil? && !secretKey.empty? - value = "secretKeyRef_#{secretName}_#{secretKey}" - end - else - value = envVar["valueFrom"].to_s - end - end - envVars.push("#{key}=#{value}") - end - envValueString = envVars.to_s - containerName = container["name"] - # Skip environment variable processing if it contains the flag AZMON_COLLECT_ENV=FALSE - # Check to see if the environment variable collection is disabled for this container. - if /AZMON_COLLECT_ENV=FALSE/i.match(envValueString) - envValueString = ["AZMON_COLLECT_ENV=FALSE"] - $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") - else - # Restricting the ENV string value to 200kb since the size of this string can go very high - if envValueString.length > 200000 - envValueStringTruncated = envValueString.slice(0..200000) - lastIndex = envValueStringTruncated.rindex("\", ") - if !lastIndex.nil? - envValueString = envValueStringTruncated.slice(0..lastIndex) + "]" - else - envValueString = envValueStringTruncated - end + response = CAdvisorMetricsAPIClient.getNodeCapacityFromCAdvisor(winNode: nil) + if !response.nil? && !response.body.nil? + cpu_capacity = JSON.parse(response.body)["num_cores"].nil? ? 1.0 : (JSON.parse(response.body)["num_cores"] * 1000.0) + memory_capacity = JSON.parse(response.body)["memory_capacity"].nil? ? 1.0 : JSON.parse(response.body)["memory_capacity"].to_f + $log.info "CPU = #{cpu_capacity}mc Memory = #{memory_capacity/1024/1024}MB" + return [cpu_capacity, memory_capacity] end - end end - rescue => error - @log.warn("KubeletUtils::parseContainerEnvironmentVarsJSON : parsing of EnvVars failed: #{error}") - end - return envValueString end - end -end +end \ No newline at end of file From a06d5c3d29e1a52cfa565ea552e6d244360d6e9c Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 6 Mar 2020 08:26:32 -0800 Subject: [PATCH 50/73] fix undefined vars --- source/code/plugin/in_containerinventory.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index e2bf7625e..f4b87aae3 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -325,9 +325,9 @@ def obtainContainerEnvironmentVars(containerId) $log.info("in_container_inventory::fetching cGroup parent filename @ #{filename}") cGroupPid = filename.split("/")[3] if @containerCGroupCache.has_key?(containerId) - tempCGroupPid = containerCgroupCache[containerId] + tempCGroupPid = containerCGroupCache[containerId] if tempCGroupPid > cGroupPid - @containerCgroupCache[containerId] = cGroupPid + @containerCGroupCache[containerId] = cGroupPid end else @containerCGroupCache[containerId] = cGroupPid From b2ea71a3074357bc6503cf0b45148d743e9a6976 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Fri, 6 Mar 2020 09:33:19 -0800 Subject: [PATCH 51/73] fix undefined vars --- source/code/plugin/in_containerinventory.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index f4b87aae3..b078573a8 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -325,7 +325,7 @@ def obtainContainerEnvironmentVars(containerId) $log.info("in_container_inventory::fetching cGroup parent filename @ #{filename}") cGroupPid = filename.split("/")[3] if @containerCGroupCache.has_key?(containerId) - tempCGroupPid = containerCGroupCache[containerId] + tempCGroupPid = @containerCGroupCache[containerId] if tempCGroupPid > cGroupPid @containerCGroupCache[containerId] = cGroupPid end @@ -343,10 +343,10 @@ def obtainContainerEnvironmentVars(containerId) # Check to see if the environment variable collection is disabled for this container. if File.foreach(environFilePath).grep(/AZMON_COLLECT_ENV=FALSE/i).any? envValueString = ["AZMON_COLLECT_ENV=FALSE"] - $log.warn("Environment Variable collection for container: #{containerName} skipped because AZMON_COLLECT_ENV is set to false") + $log.warn("Environment Variable collection for container: #{containerId} skipped because AZMON_COLLECT_ENV is set to false") else fileSize = File.size(environFilePath) - $log.info("in_container_inventory::environment vars filename @ #{filename} filesize @ #{fileSize}") + $log.info("in_container_inventory::environment vars filename @ #{environFilePath} filesize @ #{fileSize}") # Restricting the ENV string value to 200kb since the size of this string can go very high envVars = File.read(environFilePath, 200000).split(" ") envValueString = envVars.to_s From 6d145a6cc25d116e4c9766a0515a82b4fd3dc212 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 7 Mar 2020 17:14:41 -0800 Subject: [PATCH 52/73] add init containers --- source/code/plugin/in_containerinventory.rb | 145 +++++++++++--------- 1 file changed, 79 insertions(+), 66 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index b078573a8..eff39c2f4 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -202,75 +202,81 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) if !podList.nil? && !podList.empty? && podList.key?("items") && !podList["items"].nil? && !podList["items"].empty? podList["items"].each do |item| containersInfoMap = getContainersInfoMap(item) + podContainersStatuses = [] + if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? + podContainersStatuses = item["status"]["containerStatuses"] + end + if !item["status"]["initContainerStatuses"].nil? && !item["status"]["initContainerStatuses"].empty? + podContainersStatuses = podContainersStatuses + item["status"]["initContainerStatuses"] + end containerInventoryRecord = {} - if !item["status"].nil? && !item["status"].empty? - if !item["status"]["containerStatuses"].nil? && !item["status"]["containerStatuses"].empty? - item["status"]["containerStatuses"].each do |containerStatus| - containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated - containerName = containerStatus["name"] - containerId = containerStatus["containerID"].split("//")[1] - containerInventoryRecord["InstanceID"] = containerId - # imagedId is of the format - repo@sha256:imageid - imageIdValue = containerStatus["imageID"] - if !imageIdValue.nil? && !imageIdValue.empty? - atLocation = imageIdValue.index("@") - if !atLocation.nil? - containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] - end + if !podContainersStatuses.empty? + podContainersStatuses.each do |containerStatus| + containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated + containerName = containerStatus["name"] + # containeId format is :// + containerId = containerStatus["containerID"].split("//")[1] + containerInventoryRecord["InstanceID"] = containerId + # imagedId is of the format - repo@sha256:imageid + imageIdValue = containerStatus["imageID"] + if !imageIdValue.nil? && !imageIdValue.empty? + atLocation = imageIdValue.index("@") + if !atLocation.nil? + containerInventoryRecord["ImageId"] = imageIdValue[(atLocation + 1)..-1] end - # image is of the format - repository/image:imagetag - imageValue = containerStatus["image"] - if !imageValue.nil? && !imageValue.empty? - # Find delimiters in the string of format repository/image:imagetag - slashLocation = imageValue.index("/") - colonLocation = imageValue.index(":") - if !colonLocation.nil? - if slashLocation.nil? - # image:imagetag - containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] - else - # repository/image:imagetag - containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] - containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] - end - containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] - end - elsif !imageIdValue.nil? && !imageIdValue.empty? - # Getting repo information from imageIdValue when no tag in ImageId - if !atLocation.nil? - containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] + end + # image is of the format - repository/image:imagetag + imageValue = containerStatus["image"] + if !imageValue.nil? && !imageValue.empty? + # Find delimiters in the string of format repository/image:imagetag + slashLocation = imageValue.index("/") + colonLocation = imageValue.index(":") + if !colonLocation.nil? + if slashLocation.nil? + # image:imagetag + containerInventoryRecord["Image"] = imageValue[0..(colonLocation - 1)] + else + # repository/image:imagetag + containerInventoryRecord["Repository"] = imageValue[0..(slashLocation - 1)] + containerInventoryRecord["Image"] = imageValue[(slashLocation + 1)..(colonLocation - 1)] end + containerInventoryRecord["ImageTag"] = imageValue[(colonLocation + 1)..-1] end - containerInventoryRecord["ExitCode"] = 0 - if !containerStatus["state"].nil? && !containerStatus["state"].empty? - containerState = containerStatus["state"] - if containerState.key?("running") - containerInventoryRecord["State"] = "Running" - containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] - elsif containerState.key?("terminated") - containerInventoryRecord["State"] = "Terminated" - containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] - containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] - containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] - elsif containerState.key?("waiting") - containerInventoryRecord["State"] = "Waiting" - end + elsif !imageIdValue.nil? && !imageIdValue.empty? + # Getting repo information from imageIdValue when no tag in ImageId + if !atLocation.nil? + containerInventoryRecord["Repository"] = imageIdValue[0..(atLocation - 1)] end - containerInfoMap = containersInfoMap[containerName] - containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] - containerInventoryRecord["Computer"] = containerInfoMap["Computer"] - containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] - containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] - containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] - containerInventoryRecord["Ports"] = containerInfoMap["Ports"] - containerInventoryRecord["Command"] = containerInfoMap["Command"] - if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 - containerInventoryRecord["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] - else - containerInventoryRecord["EnvironmentVar"] = obtainContainerEnvironmentVars(containerId) + end + containerInventoryRecord["ExitCode"] = 0 + if !containerStatus["state"].nil? && !containerStatus["state"].empty? + containerState = containerStatus["state"] + if containerState.key?("running") + containerInventoryRecord["State"] = "Running" + containerInventoryRecord["StartedTime"] = containerState["running"]["startedAt"] + elsif containerState.key?("terminated") + containerInventoryRecord["State"] = "Terminated" + containerInventoryRecord["StartedTime"] = containerState["terminated"]["startedAt"] + containerInventoryRecord["FinishedTime"] = containerState["terminated"]["finishedAt"] + containerInventoryRecord["ExitCode"] = containerState["terminated"]["exitCode"] + elsif containerState.key?("waiting") + containerInventoryRecord["State"] = "Waiting" end - containerInventoryRecords.push containerInventoryRecord end + containerInfoMap = containersInfoMap[containerName] + containerInventoryRecord["ElementName"] = containerInfoMap["ElementName"] + containerInventoryRecord["Computer"] = containerInfoMap["Computer"] + containerInventoryRecord["ContainerHostname"] = containerInfoMap["ContainerHostname"] + containerInventoryRecord["CreatedTime"] = containerInfoMap["CreatedTime"] + containerInventoryRecord["EnvironmentVar"] = containerInfoMap["EnvironmentVar"] + containerInventoryRecord["Ports"] = containerInfoMap["Ports"] + containerInventoryRecord["Command"] = containerInfoMap["Command"] + if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 + containerInventoryRecord["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] + else + containerInventoryRecord["EnvironmentVar"] = obtainContainerEnvironmentVars(containerId) + end + containerInventoryRecords.push containerInventoryRecord end end end @@ -288,8 +294,15 @@ def getContainersInfoMap(item) nodeName = (!item["spec"]["nodeName"].nil?) ? item["spec"]["nodeName"] : "" createdTime = item["metadata"]["creationTimestamp"] if !item.nil? && !item.empty? && item.key?("spec") && !item["spec"].nil? && !item["spec"].empty? + podContainers = [] if !item["spec"]["containers"].nil? && !item["spec"]["containers"].empty? - item["spec"]["containers"].each do |container| + podContainers = item["spec"]["containers"] + end + if !item["spec"]["initContainers"].nil? && !item["spec"]["initContainers"].empty? + podContainers = podContainers + item["spec"]["initContainers"] + end + if !podContainers.empty? + podContainers.each.each do |container| containerInfoMap = {} containerName = container["name"] containerInfoMap["ElementName"] = containerName @@ -345,12 +358,12 @@ def obtainContainerEnvironmentVars(containerId) envValueString = ["AZMON_COLLECT_ENV=FALSE"] $log.warn("Environment Variable collection for container: #{containerId} skipped because AZMON_COLLECT_ENV is set to false") else - fileSize = File.size(environFilePath) - $log.info("in_container_inventory::environment vars filename @ #{environFilePath} filesize @ #{fileSize}") # Restricting the ENV string value to 200kb since the size of this string can go very high envVars = File.read(environFilePath, 200000).split(" ") envValueString = envVars.to_s - if fileSize > 200000 + envValueStringLength = envValueString.length + $log.info("in_container_inventory::environment vars filename @ #{environFilePath} envVars size @ #{envValueStringLength}") + if envValueStringLength >= 200000 lastIndex = envValueString.rindex("\", ") if !lastIndex.nil? envValueStringTruncated = envValueString.slice(0..lastIndex) + "]" From fcb47e5601a17a2bf2320763a1cd61544a564db2 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sat, 7 Mar 2020 21:17:27 -0800 Subject: [PATCH 53/73] split on null character --- source/code/plugin/in_containerinventory.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index eff39c2f4..5ac6e70c1 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -359,7 +359,7 @@ def obtainContainerEnvironmentVars(containerId) $log.warn("Environment Variable collection for container: #{containerId} skipped because AZMON_COLLECT_ENV is set to false") else # Restricting the ENV string value to 200kb since the size of this string can go very high - envVars = File.read(environFilePath, 200000).split(" ") + envVars = File.read(environFilePath, 200000).split("\0") envValueString = envVars.to_s envValueStringLength = envValueString.length $log.info("in_container_inventory::environment vars filename @ #{environFilePath} envVars size @ #{envValueStringLength}") From 69c945cf01b5b0548e1b00f7bb941005fa805552 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 8 Mar 2020 13:25:01 -0700 Subject: [PATCH 54/73] consider crio containers main proceses for envvars --- source/code/plugin/in_containerinventory.rb | 29 ++++++++++++++------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 5ac6e70c1..d5105bee5 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -215,6 +215,7 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated containerName = containerStatus["name"] # containeId format is :// + containerRuntime = containerStatus["containerID"].split(":")[0] containerId = containerStatus["containerID"].split("//")[1] containerInventoryRecord["InstanceID"] = containerId # imagedId is of the format - repo@sha256:imageid @@ -274,7 +275,12 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 containerInventoryRecord["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] else - containerInventoryRecord["EnvironmentVar"] = obtainContainerEnvironmentVars(containerId) + if !containerRuntime.nil? && !containerRuntime.empty? && containerRuntime.casecmp("cri-o") == 0 + # crio containers have conmon as parent process and we only to need get container main process envvars + containerInventoryRecord["EnvironmentVar"] = obtainContainerEnvironmentVars("crio-#{containerId}") + else + containerInventoryRecord["EnvironmentVar"] = obtainContainerEnvironmentVars(containerId) + end end containerInventoryRecords.push containerInventoryRecord end @@ -359,15 +365,18 @@ def obtainContainerEnvironmentVars(containerId) $log.warn("Environment Variable collection for container: #{containerId} skipped because AZMON_COLLECT_ENV is set to false") else # Restricting the ENV string value to 200kb since the size of this string can go very high - envVars = File.read(environFilePath, 200000).split("\0") - envValueString = envVars.to_s - envValueStringLength = envValueString.length - $log.info("in_container_inventory::environment vars filename @ #{environFilePath} envVars size @ #{envValueStringLength}") - if envValueStringLength >= 200000 - lastIndex = envValueString.rindex("\", ") - if !lastIndex.nil? - envValueStringTruncated = envValueString.slice(0..lastIndex) + "]" - envValueString = envValueStringTruncated + envVars = File.read(environFilePath, 200000) + if !envVars.nil? && !envVars.empty? + envVars = envVars.split("\0") + envValueString = envVars.to_s + envValueStringLength = envValueString.length + $log.info("in_container_inventory::environment vars filename @ #{environFilePath} envVars size @ #{envValueStringLength}") + if envValueStringLength >= 200000 + lastIndex = envValueString.rindex("\", ") + if !lastIndex.nil? + envValueStringTruncated = envValueString.slice(0..lastIndex) + "]" + envValueString = envValueStringTruncated + end end end end From 44b7374bf8de4dc08057bca02feb26cdeb2c4c1d Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 8 Mar 2020 15:16:54 -0700 Subject: [PATCH 55/73] unschedule pods or containers with pull issues --- source/code/plugin/in_containerinventory.rb | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index d5105bee5..ff7a367a2 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -215,9 +215,16 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) containerInventoryRecord["CollectionTime"] = batchTime #This is the time that is mapped to become TimeGenerated containerName = containerStatus["name"] # containeId format is :// - containerRuntime = containerStatus["containerID"].split(":")[0] - containerId = containerStatus["containerID"].split("//")[1] - containerInventoryRecord["InstanceID"] = containerId + containerRuntime = "" + containerId = "" + if !container["containerID"].nil? + containerRuntime = containerStatus["containerID"].split(":")[0] + containerId = containerStatus["containerID"].split("//")[1] + containerInventoryRecord["InstanceID"] = containerId + else + # for containers that have image issues (like invalid image/tag etc..) this will be empty. do not make it all 0 + containerInventoryRecord["InstanceID"] = containerId + end # imagedId is of the format - repo@sha256:imageid imageIdValue = containerStatus["imageID"] if !imageIdValue.nil? && !imageIdValue.empty? @@ -275,7 +282,9 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 containerInventoryRecord["EnvironmentVar"] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] else - if !containerRuntime.nil? && !containerRuntime.empty? && containerRuntime.casecmp("cri-o") == 0 + if containerId.nil? || containerId.empty? || containerRuntime.nil? || containerRuntime.empty? + containerInventoryRecord["EnvironmentVar"] = "" + elsif containerRuntime.casecmp("cri-o") == 0 # crio containers have conmon as parent process and we only to need get container main process envvars containerInventoryRecord["EnvironmentVar"] = obtainContainerEnvironmentVars("crio-#{containerId}") else @@ -308,7 +317,7 @@ def getContainersInfoMap(item) podContainers = podContainers + item["spec"]["initContainers"] end if !podContainers.empty? - podContainers.each.each do |container| + podContainers.each do |container| containerInfoMap = {} containerName = container["name"] containerInfoMap["ElementName"] = containerName From 195a919fe11ca062c8e54c74ed66c874e857bdb9 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 8 Mar 2020 15:18:52 -0700 Subject: [PATCH 56/73] fix issue --- source/code/plugin/in_containerinventory.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index ff7a367a2..0278ca0a8 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -217,7 +217,7 @@ def getContainerInventoryRecords(batchTime, clusterCollectEnvironmentVar) # containeId format is :// containerRuntime = "" containerId = "" - if !container["containerID"].nil? + if !containerStatus["containerID"].nil? containerRuntime = containerStatus["containerID"].split(":")[0] containerId = containerStatus["containerID"].split("//")[1] containerInventoryRecord["InstanceID"] = containerId From ed364dd64d7abdfaee17878da6d725fbf4762c83 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 9 Mar 2020 23:01:19 -0700 Subject: [PATCH 57/73] fix feedback --- source/code/go/src/plugins/oms.go | 5 ++++- source/code/plugin/in_kube_nodes.rb | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 0713416c6..3f128c78a 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -979,7 +979,10 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { Log("ResourceName=%s", ResourceName) } - containerRuntime = os.Getenv(ContainerRuntimeEnv) + containerRuntime = "docker" + if os.Getenv(ContainerRuntimeEnv) != "" { + containerRuntime = os.Getenv(ContainerRuntimeEnv) + } Log("Container Runtime engine %s", containerRuntime) isDockerContainerRuntimeEngine = false if strings.EqualFold(containerRuntime, "docker") { diff --git a/source/code/plugin/in_kube_nodes.rb b/source/code/plugin/in_kube_nodes.rb index 027f2d077..c5c86e640 100644 --- a/source/code/plugin/in_kube_nodes.rb +++ b/source/code/plugin/in_kube_nodes.rb @@ -192,7 +192,7 @@ def parse_and_emit_records(nodeInventory, batchTime = Time.utc.iso8601) properties["Computer"] = record["Computer"] properties["KubeletVersion"] = record["KubeletVersion"] properties["OperatingSystem"] = nodeInfo["operatingSystem"] - properties["ContainerRuntimeVersion"] = containerRuntimeVersion + # DockerVersion field holds docker version if runtime is docker/moby else :// if containerRuntimeVersion.downcase.start_with?("docker://") properties["DockerVersion"] = containerRuntimeVersion.split("//")[1] else @@ -239,7 +239,7 @@ def parse_and_emit_records(nodeInventory, batchTime = Time.utc.iso8601) if telemetrySent == true @@nodeTelemetryTimeTracker = DateTime.now.to_time.to_i end - + if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0 && eventStream.count > 0) $log.info("kubeNodeInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") end @@ -290,7 +290,7 @@ def parse_and_emit_records(nodeInventory, batchTime = Time.utc.iso8601) $log.warn "Failed when processing GPU metrics in_kube_nodes : #{errorStr}" $log.debug_backtrace(errorStr.backtrace) ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) - end + end #end GPU InsightsMetrics items rescue => errorStr $log.warn "Failed in enumerate for KubePerf from in_kube_nodes : #{errorStr}" From 3a5d0ec72298624862beac33a5ccd3e1e20363da Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 10 Mar 2020 17:10:00 -0700 Subject: [PATCH 58/73] update to azmon parsers --- .../conf/azm-containers-parser.conf.conf | 17 +++++++++++ installer/conf/td-agent-bit-rs.conf | 2 +- installer/conf/td-agent-bit.conf | 2 +- installer/datafiles/base_container.data | 1 + source/code/go/src/plugins/oms.go | 29 ++++--------------- 5 files changed, 25 insertions(+), 26 deletions(-) create mode 100644 installer/conf/azm-containers-parser.conf.conf diff --git a/installer/conf/azm-containers-parser.conf.conf b/installer/conf/azm-containers-parser.conf.conf new file mode 100644 index 000000000..d0335fba8 --- /dev/null +++ b/installer/conf/azm-containers-parser.conf.conf @@ -0,0 +1,17 @@ +[PARSER] + Name docker + Format json + Time_Key time + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep On + # Command | Decoder | Field | Optional Action + # =============|==================|================= + Decode_Field_As escaped log + +[PARSER] + # http://rubular.com/r/tjUt3Awgg4 + Name cri + Format regex + Regex ^(?