From 8a3e046e5fadd673323fbbe0f3b4a918d127a46e Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 30 Sep 2019 15:44:25 -0700 Subject: [PATCH 01/20] init container - KPI and kubeperf changes --- source/code/plugin/KubernetesApiClient.rb | 15 ++++++++++-- source/code/plugin/in_kube_podinventory.rb | 27 ++++++++++++++++++---- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/source/code/plugin/KubernetesApiClient.rb b/source/code/plugin/KubernetesApiClient.rb index 48b25bf14..68ea35533 100644 --- a/source/code/plugin/KubernetesApiClient.rb +++ b/source/code/plugin/KubernetesApiClient.rb @@ -356,9 +356,20 @@ def getContainerResourceRequestsAndLimits(metricJSON, metricCategory, metricName else podUid = pod["metadata"]["uid"] end - if (!pod["spec"]["containers"].nil? && !pod["spec"]["nodeName"].nil?) + + podContainers = [] + if !pod["spec"]["containers"].nil? && !pod["spec"]["containers"].empty? + podContainers = podContainers + pod["spec"]["containers"] + end + # Adding init containers to the record list as well. + if !pod["spec"]["initContainers"].nil? && !pod["spec"]["initContainers"].empty? + podContainers = podContainers + pod["spec"]["initContainers"] + end + + # if (!pod["spec"]["containers"].nil? && !pod["spec"]["nodeName"].nil?) + if (!podContainers.nil? && !podContainers.empty? && !pod["spec"]["nodeName"].nil?) nodeName = pod["spec"]["nodeName"] - pod["spec"]["containers"].each do |container| + podContainers.each do |container| containerName = container["name"] metricTime = Time.now.utc.iso8601 #2018-01-30T19:36:14Z if (!container["resources"].nil? && !container["resources"].empty? && !container["resources"][metricCategory].nil? && !container["resources"][metricCategory][metricNameToCollect].nil?) diff --git a/source/code/plugin/in_kube_podinventory.rb b/source/code/plugin/in_kube_podinventory.rb index f41ce9095..60e0b3804 100644 --- a/source/code/plugin/in_kube_podinventory.rb +++ b/source/code/plugin/in_kube_podinventory.rb @@ -137,8 +137,16 @@ def getContainerEnvironmentVariables(pod, clusterCollectEnvironmentVar) begin podSpec = pod["spec"] containerEnvHash = {} - if !podSpec.nil? && !podSpec["containers"].nil? - podSpec["containers"].each do |container| + podContainersEnv = [] + if !podSpec["containers"].nil? && !podSpec["containers"].empty? + podContainersEnv = podContainersEnv + podSpec["containers"] + end + # Adding init containers to the record list as well. + if !podSpec["initContainers"].nil? && !podSpec["initContainers"].empty? + podContainersEnv = podContainersEnv + podSpec["initContainers"] + end + if !podContainersEnv.nil? && !podContainersEnv.empty? + podContainersEnv.each do |container| if !clusterCollectEnvironmentVar.nil? && !clusterCollectEnvironmentVar.empty? && clusterCollectEnvironmentVar.casecmp("false") == 0 containerEnvHash[container["name"]] = ["AZMON_CLUSTER_COLLECT_ENV_VAR=FALSE"] else @@ -289,8 +297,19 @@ def parse_and_emit_records(podInventory, serviceList) end podRestartCount = 0 record["PodRestartCount"] = 0 - if items["status"].key?("containerStatuses") && !items["status"]["containerStatuses"].empty? #container status block start - items["status"]["containerStatuses"].each do |container| + + podContainers = [] + if items["status"].key?("containerStatuses") && !items["status"]["containerStatuses"].empty? + podContainers = podContainers + items["status"]["containerStatuses"] + end + # Adding init containers to the record list as well. + if items["status"].key?("initContainerStatuses") && !items["status"]["initContainerStatuses"].empty? + podContainers = podContainers + items["status"]["initContainerStatuses"] + end + + # if items["status"].key?("containerStatuses") && !items["status"]["containerStatuses"].empty? #container status block start + if !podContainers.empty? #container status block start + podContainers.each do |container| containerRestartCount = 0 #container Id is of the form #docker://dfd9da983f1fd27432fb2c1fe3049c0a1d25b1c697b2dc1a530c986e58b16527 From 49f14c01c000e3eb6f305c9cb85b938e49edc6ba Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 30 Sep 2019 16:14:53 -0700 Subject: [PATCH 02/20] changes --- source/code/go/src/plugins/oms.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index c5ad307d8..49f0ecffb 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -195,7 +195,14 @@ func updateContainerImageNameMaps() { } for _, pod := range pods.Items { - for _, status := range pod.Status.ContainerStatuses { + // Doing this to include init container logs as well + podContainerStatuses, err := pod.Status.ContainerStatuses + + podInitContainerStatuses, err := pod.Status.InitContainerStatuses + if (podInitContainerStatuses != nil) && (len(podInitContainerStatuses) > 0) { + podContainerStatuses = append(podContainerStatuses, podInitContainerStatuses...) + } + for _, status := range podContainerStatuses { lastSlashIndex := strings.LastIndex(status.ContainerID, "/") containerID := status.ContainerID[lastSlashIndex+1 : len(status.ContainerID)] image := status.Image From 4600431bb0f0eb3dba7a212c0f6fb4629da87eaf Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 30 Sep 2019 16:32:49 -0700 Subject: [PATCH 03/20] changes --- source/code/go/src/plugins/oms.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 49f0ecffb..1312d480d 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -195,10 +195,11 @@ func updateContainerImageNameMaps() { } for _, pod := range pods.Items { + podContainerStatuses := pod.Status.ContainerStatuses + // Doing this to include init container logs as well - podContainerStatuses, err := pod.Status.ContainerStatuses + podInitContainerStatuses := pod.Status.InitContainerStatuses - podInitContainerStatuses, err := pod.Status.InitContainerStatuses if (podInitContainerStatuses != nil) && (len(podInitContainerStatuses) > 0) { podContainerStatuses = append(podContainerStatuses, podInitContainerStatuses...) } From 387f9a79e70aa127b1186623732f89a025803446 Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 30 Sep 2019 16:46:07 -0700 Subject: [PATCH 04/20] changes --- source/code/go/src/plugins/oms.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 1312d480d..b5e2ef309 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -199,7 +199,7 @@ func updateContainerImageNameMaps() { // Doing this to include init container logs as well podInitContainerStatuses := pod.Status.InitContainerStatuses - + Log("Length of init containers %d", len(podInitContainerStatuses)) if (podInitContainerStatuses != nil) && (len(podInitContainerStatuses) > 0) { podContainerStatuses = append(podContainerStatuses, podInitContainerStatuses...) } From 956ea77c727ab78a55a9dc07ec92e468780bf46b Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 1 Oct 2019 11:28:46 -0700 Subject: [PATCH 05/20] changes for empty array fix --- installer/scripts/tomlparser-prom-customconfig.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/scripts/tomlparser-prom-customconfig.rb b/installer/scripts/tomlparser-prom-customconfig.rb index d44bf3342..c5960fc5b 100644 --- a/installer/scripts/tomlparser-prom-customconfig.rb +++ b/installer/scripts/tomlparser-prom-customconfig.rb @@ -46,7 +46,7 @@ def parseConfigMap end def checkForTypeArray(arrayValue, arrayType) - if (arrayValue.nil? || (arrayValue.kind_of?(Array) && arrayValue.length > 0 && arrayValue[0].kind_of?(arrayType))) + if (arrayValue.nil? || (arrayValue.kind_of?(Array) && ((arrayValue.length == 0) || (arrayValue.length > 0 && arrayValue[0].kind_of?(arrayType)))) return true else return false From b6759fb61e89d5784834e05b9b949fd85968876d Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 1 Oct 2019 11:40:10 -0700 Subject: [PATCH 06/20] changes --- installer/scripts/tomlparser-prom-customconfig.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/scripts/tomlparser-prom-customconfig.rb b/installer/scripts/tomlparser-prom-customconfig.rb index c5960fc5b..f0b616b2e 100644 --- a/installer/scripts/tomlparser-prom-customconfig.rb +++ b/installer/scripts/tomlparser-prom-customconfig.rb @@ -46,7 +46,7 @@ def parseConfigMap end def checkForTypeArray(arrayValue, arrayType) - if (arrayValue.nil? || (arrayValue.kind_of?(Array) && ((arrayValue.length == 0) || (arrayValue.length > 0 && arrayValue[0].kind_of?(arrayType)))) + if (arrayValue.nil? || (arrayValue.kind_of?(Array) && ((arrayValue.length == 0) || (arrayValue.length > 0 && arrayValue[0].kind_of?(arrayType))))) return true else return false From 4820647afb00833c7e627d0aa144d16e2d52ad5d Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 1 Oct 2019 13:35:29 -0700 Subject: [PATCH 07/20] changes --- installer/conf/telegraf.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/installer/conf/telegraf.conf b/installer/conf/telegraf.conf index 4883de81b..b2a54fc9a 100644 --- a/installer/conf/telegraf.conf +++ b/installer/conf/telegraf.conf @@ -532,7 +532,8 @@ name_prefix="container.azm.ms/" ## An array of urls to scrape metrics from. urls = ["http://$NODE_IP:10255/metrics"] - fieldpass = ["kubelet_docker_operations", "kubelet_docker_operations_errors"] + #fieldpass = ["kubelet_docker_operations", "kubelet_docker_operations_errors"] + fieldpass = ["kubelet_docker_operations_errors"] metric_version = 2 url_tag = "scrapeUrl" From 005b82bc43dd4ed3eb23bae510cdb8292c5e9a70 Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 2 Oct 2019 18:43:16 -0700 Subject: [PATCH 08/20] pod inventory exception fix --- source/code/plugin/in_kube_podinventory.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/source/code/plugin/in_kube_podinventory.rb b/source/code/plugin/in_kube_podinventory.rb index 60e0b3804..766831a66 100644 --- a/source/code/plugin/in_kube_podinventory.rb +++ b/source/code/plugin/in_kube_podinventory.rb @@ -48,13 +48,15 @@ def shutdown end def enumerate(podList = nil) - if podList.nil? - $log.info("in_kube_podinventory::enumerate : Getting pods from Kube API @ #{Time.now.utc.iso8601}") - podInventory = JSON.parse(KubernetesApiClient.getKubeResourceInfo("pods").body) - $log.info("in_kube_podinventory::enumerate : Done getting pods from Kube API @ #{Time.now.utc.iso8601}") - else - podInventory = podList + podInventory = podList + $log.info("in_kube_podinventory::enumerate : Getting pods from Kube API @ #{Time.now.utc.iso8601}") + podInfo = KubernetesApiClient.getKubeResourceInfo("pods") + $log.info("in_kube_podinventory::enumerate : Done getting pods from Kube API @ #{Time.now.utc.iso8601}") + + if !podInfo.nil? + podInventory = JSON.parse(podInfo.body) end + begin if (!podInventory.empty? && podInventory.key?("items") && !podInventory["items"].empty?) #get pod inventory & services From 3407715321a1df1f896f4f913e7350f4583463ab Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 2 Oct 2019 19:04:14 -0700 Subject: [PATCH 09/20] nil check changes --- installer/conf/telegraf.conf | 1 - source/code/plugin/in_kube_events.rb | 17 ++++++++++------- source/code/plugin/in_kube_services.rb | 12 ++++++++++-- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/installer/conf/telegraf.conf b/installer/conf/telegraf.conf index b2a54fc9a..271e7dd07 100644 --- a/installer/conf/telegraf.conf +++ b/installer/conf/telegraf.conf @@ -532,7 +532,6 @@ name_prefix="container.azm.ms/" ## An array of urls to scrape metrics from. urls = ["http://$NODE_IP:10255/metrics"] - #fieldpass = ["kubelet_docker_operations", "kubelet_docker_operations_errors"] fieldpass = ["kubelet_docker_operations_errors"] metric_version = 2 diff --git a/source/code/plugin/in_kube_events.rb b/source/code/plugin/in_kube_events.rb index f177b62bf..0e3769709 100644 --- a/source/code/plugin/in_kube_events.rb +++ b/source/code/plugin/in_kube_events.rb @@ -47,17 +47,20 @@ def enumerate(eventList = nil) currentTime = Time.now emitTime = currentTime.to_f batchTime = currentTime.utc.iso8601 - if eventList.nil? - $log.info("in_kube_events::enumerate : Getting events from Kube API @ #{Time.now.utc.iso8601}") - events = JSON.parse(KubernetesApiClient.getKubeResourceInfo("events").body) - $log.info("in_kube_events::enumerate : Done getting events from Kube API @ #{Time.now.utc.iso8601}") - else - events = eventList + + events = eventList + $log.info("in_kube_events::enumerate : Getting events from Kube API @ #{Time.now.utc.iso8601}") + eventInfo = KubernetesApiClient.getKubeResourceInfo("events") + $log.info("in_kube_events::enumerate : Done getting events from Kube API @ #{Time.now.utc.iso8601}") + + if !eventInfo.nil? + events = JSON.parse(eventInfo.body) end + eventQueryState = getEventQueryState newEventQueryState = [] begin - if (!events.empty? && !events["items"].nil?) + if (!events.nil? && !events.empty? && !events["items"].nil?) eventStream = MultiEventStream.new events["items"].each do |items| record = {} diff --git a/source/code/plugin/in_kube_services.rb b/source/code/plugin/in_kube_services.rb index 8b0a013e4..7cd703620 100644 --- a/source/code/plugin/in_kube_services.rb +++ b/source/code/plugin/in_kube_services.rb @@ -46,11 +46,19 @@ def enumerate currentTime = Time.now emitTime = currentTime.to_f batchTime = currentTime.utc.iso8601 + + serviceList = nil + $log.info("in_kube_services::enumerate : Getting services from Kube API @ #{Time.now.utc.iso8601}") - serviceList = JSON.parse(KubernetesApiClient.getKubeResourceInfo("services").body) + serviceInfo = KubernetesApiClient.getKubeResourceInfo("services") $log.info("in_kube_services::enumerate : Done getting services from Kube API @ #{Time.now.utc.iso8601}") + + if !serviceInfo.nil? + serviceList = JSON.parse(serviceInfo.body) + end + begin - if (!serviceList.empty?) + if (!serviceList.nil? && !serviceList.empty?) eventStream = MultiEventStream.new serviceList["items"].each do |items| record = {} From f7c9299a144aabf8fb147e9f45c42f0fafe3737d Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 2 Oct 2019 21:41:59 -0700 Subject: [PATCH 10/20] changes --- source/code/plugin/in_kube_nodes.rb | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/source/code/plugin/in_kube_nodes.rb b/source/code/plugin/in_kube_nodes.rb index 42bc13b68..0a0fd9d2e 100644 --- a/source/code/plugin/in_kube_nodes.rb +++ b/source/code/plugin/in_kube_nodes.rb @@ -61,11 +61,19 @@ def enumerate emitTime = currentTime.to_f batchTime = currentTime.utc.iso8601 telemetrySent = false + + nodeInventory = nil + $log.info("in_kube_nodes::enumerate : Getting nodes from Kube API @ #{Time.now.utc.iso8601}") - nodeInventory = JSON.parse(KubernetesApiClient.getKubeResourceInfo("nodes").body) + nodeInfo = KubernetesApiClient.getKubeResourceInfo("nodes") $log.info("in_kube_nodes::enumerate : Done getting nodes from Kube API @ #{Time.now.utc.iso8601}") + + if !nodeInfo.nil? + nodeInventory = JSON.parse(nodeInfo.body) + end + begin - if (!nodeInventory.empty?) + if (!nodeInventory.nil? && !nodeInventory.empty?) eventStream = MultiEventStream.new containerNodeInventoryEventStream = MultiEventStream.new if !nodeInventory["items"].nil? @@ -95,7 +103,6 @@ def enumerate record["KubernetesProviderID"] = "onprem" end - # Refer to https://kubernetes.io/docs/concepts/architecture/nodes/#condition for possible node conditions. # We check the status of each condition e.g. {"type": "OutOfDisk","status": "False"} . Based on this we # populate the KubeNodeInventory Status field. A possible value for this field could be "Ready OutofDisk" From 9c5824a8574bff0bef1acaf5c2c2996128f13913 Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 3 Oct 2019 15:21:49 -0700 Subject: [PATCH 11/20] fixing typo --- source/code/go/src/plugins/oms.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 808d882a1..78d36a92c 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -186,8 +186,8 @@ type laKubeMonAgentEvents struct { type KubeMonAgentEventTags struct { PodName string ContainerId string - FirstOccurance string - LastOccurance string + FirstOccurrence string + LastOccurrence string Count int } @@ -352,21 +352,21 @@ func populateKubeMonAgentEventHash(record map[interface{}]interface{}, errType K if val, ok := ConfigErrorEvent[logRecordString]; ok { Log("In config error existing hash update\n") eventCount := val.Count - eventFirstOccurance := val.FirstOccurance + eventFirstOccurrence := val.FirstOccurrence ConfigErrorEvent[logRecordString] = KubeMonAgentEventTags{ PodName: podName, ContainerId: containerID, - FirstOccurance: eventFirstOccurance, - LastOccurance: eventTimeStamp, + FirstOccurrence: eventFirstOccurrence, + LastOccurrence: eventTimeStamp, Count: eventCount + 1, } } else { ConfigErrorEvent[logRecordString] = KubeMonAgentEventTags{ PodName: podName, ContainerId: containerID, - FirstOccurance: eventTimeStamp, - LastOccurance: eventTimeStamp, + FirstOccurrence: eventTimeStamp, + LastOccurrence: eventTimeStamp, Count: 1, } } @@ -382,21 +382,21 @@ func populateKubeMonAgentEventHash(record map[interface{}]interface{}, errType K if val, ok := PromScrapeErrorEvent[splitString]; ok { Log("In config error existing hash update\n") eventCount := val.Count - eventFirstOccurance := val.FirstOccurance + eventFirstOccurrence := val.FirstOccurrence PromScrapeErrorEvent[splitString] = KubeMonAgentEventTags{ PodName: podName, ContainerId: containerID, - FirstOccurance: eventFirstOccurance, - LastOccurance: eventTimeStamp, + FirstOccurrence: eventFirstOccurrence, + LastOccurrence: eventTimeStamp, Count: eventCount + 1, } } else { PromScrapeErrorEvent[splitString] = KubeMonAgentEventTags{ PodName: podName, ContainerId: containerID, - FirstOccurance: eventTimeStamp, - LastOccurance: eventTimeStamp, + FirstOccurrence: eventTimeStamp, + LastOccurrence: eventTimeStamp, Count: 1, } } From 47c80d55d4505da014b3288efa13608eaba4bd74 Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 3 Oct 2019 16:04:54 -0700 Subject: [PATCH 12/20] changes --- source/code/go/src/plugins/oms.go | 51 ++++++++++++++++--------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 78d36a92c..1a5367ca7 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -184,11 +184,11 @@ type laKubeMonAgentEvents struct { } type KubeMonAgentEventTags struct { - PodName string - ContainerId string + PodName string + ContainerId string FirstOccurrence string LastOccurrence string - Count int + Count int } type KubeMonAgentEventBlob struct { @@ -355,19 +355,19 @@ func populateKubeMonAgentEventHash(record map[interface{}]interface{}, errType K eventFirstOccurrence := val.FirstOccurrence ConfigErrorEvent[logRecordString] = KubeMonAgentEventTags{ - PodName: podName, - ContainerId: containerID, + PodName: podName, + ContainerId: containerID, FirstOccurrence: eventFirstOccurrence, LastOccurrence: eventTimeStamp, - Count: eventCount + 1, + Count: eventCount + 1, } } else { ConfigErrorEvent[logRecordString] = KubeMonAgentEventTags{ - PodName: podName, - ContainerId: containerID, + PodName: podName, + ContainerId: containerID, FirstOccurrence: eventTimeStamp, LastOccurrence: eventTimeStamp, - Count: 1, + Count: 1, } } @@ -385,19 +385,19 @@ func populateKubeMonAgentEventHash(record map[interface{}]interface{}, errType K eventFirstOccurrence := val.FirstOccurrence PromScrapeErrorEvent[splitString] = KubeMonAgentEventTags{ - PodName: podName, - ContainerId: containerID, + PodName: podName, + ContainerId: containerID, FirstOccurrence: eventFirstOccurrence, LastOccurrence: eventTimeStamp, - Count: eventCount + 1, + Count: eventCount + 1, } } else { PromScrapeErrorEvent[splitString] = KubeMonAgentEventTags{ - PodName: podName, - ContainerId: containerID, + PodName: podName, + ContainerId: containerID, FirstOccurrence: eventTimeStamp, LastOccurrence: eventTimeStamp, - Count: 1, + Count: 1, } } } @@ -764,16 +764,17 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { FlushedRecordsSize += float64(len(stringMap["LogEntry"])) dataItems = append(dataItems, dataItem) - loggedTime, e := time.Parse(time.RFC3339, dataItem.LogEntryTimeStamp) - if e != nil { - message := fmt.Sprintf("Error while converting LogEntryTimeStamp for telemetry purposes: %s", e.Error()) - Log(message) - SendException(message) - } else { - ltncy := float64(start.Sub(loggedTime) / time.Millisecond) - if ltncy >= maxLatency { - maxLatency = ltncy - maxLatencyContainer = dataItem.Name + "=" + dataItem.ID + if (dataItem.LogEntryTimeStamp != nil) && (dataItem.LogEntryTimeStamp != "") { + loggedTime, e := time.Parse(time.RFC3339, dataItem.LogEntryTimeStamp) + if e != nil { + message := fmt.Sprintf("Error while converting LogEntryTimeStamp for telemetry purposes: %s", e.Error()) + Log(message) + } else { + ltncy := float64(start.Sub(loggedTime) / time.Millisecond) + if ltncy >= maxLatency { + maxLatency = ltncy + maxLatencyContainer = dataItem.Name + "=" + dataItem.ID + } } } } From 5014ec8b13cb947961ceef93a9662118f8c29806 Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 3 Oct 2019 16:06:31 -0700 Subject: [PATCH 13/20] changes --- source/code/go/src/plugins/oms.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 1a5367ca7..09f2592b5 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -764,7 +764,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { FlushedRecordsSize += float64(len(stringMap["LogEntry"])) dataItems = append(dataItems, dataItem) - if (dataItem.LogEntryTimeStamp != nil) && (dataItem.LogEntryTimeStamp != "") { + if dataItem.LogEntryTimeStamp != "" { loggedTime, e := time.Parse(time.RFC3339, dataItem.LogEntryTimeStamp) if e != nil { message := fmt.Sprintf("Error while converting LogEntryTimeStamp for telemetry purposes: %s", e.Error()) From 06c555d10041369261e2ec9df298e341517ae6f0 Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 3 Oct 2019 17:47:05 -0700 Subject: [PATCH 14/20] PR - feedback --- installer/conf/telegraf.conf | 3 ++- source/code/go/src/plugins/oms.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/installer/conf/telegraf.conf b/installer/conf/telegraf.conf index 271e7dd07..8c4edca3b 100644 --- a/installer/conf/telegraf.conf +++ b/installer/conf/telegraf.conf @@ -532,7 +532,8 @@ name_prefix="container.azm.ms/" ## An array of urls to scrape metrics from. urls = ["http://$NODE_IP:10255/metrics"] - fieldpass = ["kubelet_docker_operations_errors"] + fieldpass = ["kubelet_docker_operations", "kubelet_docker_operations_errors"] + taginclude = ["create_container","remove_container", "pull_image"] metric_version = 2 url_tag = "scrapeUrl" diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 09f2592b5..6a348085a 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -769,6 +769,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { if e != nil { message := fmt.Sprintf("Error while converting LogEntryTimeStamp for telemetry purposes: %s", e.Error()) Log(message) + SendException(message) } else { ltncy := float64(start.Sub(loggedTime) / time.Millisecond) if ltncy >= maxLatency { From 267781ee8bde56873014576e30bb68c3fb2b59f0 Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 3 Oct 2019 17:57:04 -0700 Subject: [PATCH 15/20] remove comment --- source/code/plugin/KubernetesApiClient.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/source/code/plugin/KubernetesApiClient.rb b/source/code/plugin/KubernetesApiClient.rb index 68ea35533..be1a51791 100644 --- a/source/code/plugin/KubernetesApiClient.rb +++ b/source/code/plugin/KubernetesApiClient.rb @@ -366,7 +366,6 @@ def getContainerResourceRequestsAndLimits(metricJSON, metricCategory, metricName podContainers = podContainers + pod["spec"]["initContainers"] end - # if (!pod["spec"]["containers"].nil? && !pod["spec"]["nodeName"].nil?) if (!podContainers.nil? && !podContainers.empty? && !pod["spec"]["nodeName"].nil?) nodeName = pod["spec"]["nodeName"] podContainers.each do |container| From c26fd3ab37ed3028e663cf11bf87ca715745c2b7 Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 4 Oct 2019 14:44:50 -0700 Subject: [PATCH 16/20] tag pass changes --- installer/conf/telegraf.conf | 4 ++-- source/code/go/src/plugins/oms.go | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/installer/conf/telegraf.conf b/installer/conf/telegraf.conf index 8c4edca3b..8da8eebcf 100644 --- a/installer/conf/telegraf.conf +++ b/installer/conf/telegraf.conf @@ -533,7 +533,6 @@ ## An array of urls to scrape metrics from. urls = ["http://$NODE_IP:10255/metrics"] fieldpass = ["kubelet_docker_operations", "kubelet_docker_operations_errors"] - taginclude = ["create_container","remove_container", "pull_image"] metric_version = 2 url_tag = "scrapeUrl" @@ -567,7 +566,8 @@ ## Use TLS but skip chain & host verification insecure_skip_verify = true #tagexclude = ["AgentVersion","AKS_RESOURCE_ID","ACS_RESOURCE_NAME", "Region", "ClusterName", "ClusterType", "Computer", "ControllerType"] - + [inputs.prometheus.tagpass] + operation_type = ["create_container", "delete_container", "pull_image"] ## prometheus custom metrics [[inputs.prometheus]] diff --git a/source/code/go/src/plugins/oms.go b/source/code/go/src/plugins/oms.go index 6a348085a..01aab85b4 100644 --- a/source/code/go/src/plugins/oms.go +++ b/source/code/go/src/plugins/oms.go @@ -263,7 +263,6 @@ func updateContainerImageNameMaps() { // Doing this to include init container logs as well podInitContainerStatuses := pod.Status.InitContainerStatuses - Log("Length of init containers %d", len(podInitContainerStatuses)) if (podInitContainerStatuses != nil) && (len(podInitContainerStatuses) > 0) { podContainerStatuses = append(podContainerStatuses, podInitContainerStatuses...) } From e150e903e4fde25df9d41af412f64cbc4a9d67a8 Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 4 Oct 2019 15:19:30 -0700 Subject: [PATCH 17/20] changes --- installer/conf/telegraf.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/installer/conf/telegraf.conf b/installer/conf/telegraf.conf index 8da8eebcf..0206d8150 100644 --- a/installer/conf/telegraf.conf +++ b/installer/conf/telegraf.conf @@ -567,7 +567,9 @@ insecure_skip_verify = true #tagexclude = ["AgentVersion","AKS_RESOURCE_ID","ACS_RESOURCE_NAME", "Region", "ClusterName", "ClusterType", "Computer", "ControllerType"] [inputs.prometheus.tagpass] - operation_type = ["create_container", "delete_container", "pull_image"] + operation_type = ["create_container", "remove_container", "pull_image"] + # [inputs.prometheus.tagdrop] + # operation_type = ["version", "stop_container", "start_container", "list_images", "list_containers", "inspect_image", "inspect_container", "info"] ## prometheus custom metrics [[inputs.prometheus]] From 677805e76aa07504697c3aa1e769341400080c9b Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 4 Oct 2019 15:32:50 -0700 Subject: [PATCH 18/20] tagdrop changes --- installer/conf/telegraf.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/installer/conf/telegraf.conf b/installer/conf/telegraf.conf index 0206d8150..8cdb301aa 100644 --- a/installer/conf/telegraf.conf +++ b/installer/conf/telegraf.conf @@ -566,10 +566,10 @@ ## Use TLS but skip chain & host verification insecure_skip_verify = true #tagexclude = ["AgentVersion","AKS_RESOURCE_ID","ACS_RESOURCE_NAME", "Region", "ClusterName", "ClusterType", "Computer", "ControllerType"] - [inputs.prometheus.tagpass] - operation_type = ["create_container", "remove_container", "pull_image"] - # [inputs.prometheus.tagdrop] - # operation_type = ["version", "stop_container", "start_container", "list_images", "list_containers", "inspect_image", "inspect_container", "info"] + # [inputs.prometheus.tagpass] + # operation_type = ["create_container", "remove_container", "pull_image"] + [inputs.prometheus.tagdrop] + operation_type = ["version", "stop_container", "start_container", "list_images", "list_containers", "inspect_image", "inspect_container", "info"] ## prometheus custom metrics [[inputs.prometheus]] From 404a206962ac6b7f63720ff10f668d2fc5a70629 Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 4 Oct 2019 15:40:24 -0700 Subject: [PATCH 19/20] changes --- installer/conf/telegraf.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/installer/conf/telegraf.conf b/installer/conf/telegraf.conf index 8cdb301aa..0206d8150 100644 --- a/installer/conf/telegraf.conf +++ b/installer/conf/telegraf.conf @@ -566,10 +566,10 @@ ## Use TLS but skip chain & host verification insecure_skip_verify = true #tagexclude = ["AgentVersion","AKS_RESOURCE_ID","ACS_RESOURCE_NAME", "Region", "ClusterName", "ClusterType", "Computer", "ControllerType"] - # [inputs.prometheus.tagpass] - # operation_type = ["create_container", "remove_container", "pull_image"] - [inputs.prometheus.tagdrop] - operation_type = ["version", "stop_container", "start_container", "list_images", "list_containers", "inspect_image", "inspect_container", "info"] + [inputs.prometheus.tagpass] + operation_type = ["create_container", "remove_container", "pull_image"] + # [inputs.prometheus.tagdrop] + # operation_type = ["version", "stop_container", "start_container", "list_images", "list_containers", "inspect_image", "inspect_container", "info"] ## prometheus custom metrics [[inputs.prometheus]] From be71945ad723bde683a5fe1d5267121015939c45 Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 4 Oct 2019 15:40:51 -0700 Subject: [PATCH 20/20] changes --- installer/conf/telegraf.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/installer/conf/telegraf.conf b/installer/conf/telegraf.conf index 0206d8150..cd22a56b4 100644 --- a/installer/conf/telegraf.conf +++ b/installer/conf/telegraf.conf @@ -568,8 +568,6 @@ #tagexclude = ["AgentVersion","AKS_RESOURCE_ID","ACS_RESOURCE_NAME", "Region", "ClusterName", "ClusterType", "Computer", "ControllerType"] [inputs.prometheus.tagpass] operation_type = ["create_container", "remove_container", "pull_image"] - # [inputs.prometheus.tagdrop] - # operation_type = ["version", "stop_container", "start_container", "list_images", "list_containers", "inspect_image", "inspect_container", "info"] ## prometheus custom metrics [[inputs.prometheus]]