Merge ci_feature to ci_feature_prod

README.md

@@ -11,6 +11,25 @@ additional questions or comments.
 
 Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate the agent build dates (not release dates)
 
+### 03/02/2020 -
+##### Version microsoft/oms:ciprod03022020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03022020
+##### Code change log
+- Collection of GPU metrics as InsightsMetrics
+- Enable config map settings to enable collection of 'Normal' kube events
+- Fix kubehealth exceptions to handle empty/nil kube api responses
+- Get resource limits for health and MDM from kubelet instead of kube api
+- Bug fix for windows node image collection where image name contains multiple slashes
+- Exclude ARO master node for data collection
+- Telemetry for kube events flushed
+- Changes to support msi for mdm if service principal doesnt exist
+- Changes for AKS telemetry to ping ods endpoint first and then network check
+- KubeEvents bug fix for KubeEvent type
+
+##### Customer Impact
+- Providing capability for customers to collect 'Normal' kube events using config map
+- Metrics for GPU are collected and ingested to customers workspace if they have GPU enabled nodes 
+- Bug fix for windows container image collection allows customers to get the right data in the ContainerInventory table for windows       containers.
+
 ### 01/07/2020 -
 ##### Version microsoft/oms:ciprod01072020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod01072020
 ##### Code change log

installer/conf/azm-containers-parser.conf

@@ -0,0 +1,19 @@
+[PARSER]
+    Name         docker
+    Format       json
+    Time_Key     time
+    Time_Format  %Y-%m-%dT%H:%M:%S.%L
+    Time_Keep    On
+    # Command      |  Decoder | Field | Optional Action
+    # =============|==================|=================
+    Decode_Field_As   escaped    log
+
+[PARSER]
+    # http://rubular.com/r/tjUt3Awgg4
+    Name cri
+    Format regex
+    Regex ^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<logtag>[^ ]*) (?<log>.*)$
+    Time_Key    time
+    Time_Format %Y-%m-%dT%H:%M:%S.%L
+    Time_Keep    On   
+

installer/conf/td-agent-bit-rs.conf

@@ -1,7 +1,7 @@
 [SERVICE]
     Flush         30
     Log_Level     info
-    Parsers_File  /etc/td-agent-bit/parsers.conf
+    Parsers_File  /etc/opt/microsoft/docker-cimprov/azm-containers-parser.conf
     Log_File      /var/opt/microsoft/docker-cimprov/log/fluent-bit.log
 
 [INPUT]

installer/conf/td-agent-bit.conf

@@ -2,7 +2,7 @@
     #Default service flush interval is 15 seconds
     ${SERVICE_FLUSH_INTERVAL}
     Log_Level     info
-    Parsers_File  /etc/td-agent-bit/parsers.conf
+    Parsers_File  /etc/opt/microsoft/docker-cimprov/azm-containers-parser.conf
     Log_File      /var/opt/microsoft/docker-cimprov/log/fluent-bit.log
 
 [INPUT]

installer/conf/telegraf-rs.conf

@@ -56,7 +56,7 @@
 
   ## Default flushing interval for all outputs. You shouldn't set this below
   ## interval. Maximum flush_interval will be flush_interval + flush_jitter
-  flush_interval = "60s"
+  flush_interval = "15s"
   ## Jitter the flush interval by a random amount. This is primarily to avoid
   ## large write spikes for users running a large number of telegraf instances.
   ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s

installer/conf/telegraf.conf

@@ -56,7 +56,7 @@
 
   ## Default flushing interval for all outputs. You shouldn't set this below
   ## interval. Maximum flush_interval will be flush_interval + flush_jitter
-  flush_interval = "60s"
+  flush_interval = "15s"
   ## Jitter the flush interval by a random amount. This is primarily to avoid
   ## large write spikes for users running a large number of telegraf instances.
   ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
@@ -386,7 +386,7 @@
 #  report_active = true
 #  fieldpass = ["usage_active","cluster","node","host","device"]
 #  taginclude = ["cluster","cpu","node"]
-  
+
 
 
 # Read metrics about disk usage by mount point
@@ -395,7 +395,7 @@
   ## By default stats will be gathered for all mount points.
   ## Set mount_points will restrict the stats to only the specified mount points.
   # mount_points = ["/"]
-  
+
   ## Ignore mount points by filesystem type.
   ignore_fs = ["tmpfs", "devtmpfs", "devfs", "overlay", "aufs", "squashfs"]
   fieldpass = ["free", "used", "used_percent"]
@@ -532,7 +532,8 @@
   name_prefix="container.azm.ms/"
   ## An array of urls to scrape metrics from.
   urls = ["$CADVISOR_METRICS_URL"]
-  fieldpass = ["kubelet_docker_operations", "kubelet_docker_operations_errors"]
+  ## Include "$KUBELET_RUNTIME_OPERATIONS_TOTAL_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_TOTAL_METRIC" when we add for support for 1.18
+  fieldpass = ["$KUBELET_RUNTIME_OPERATIONS_METRIC", "$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC"]
 
   metric_version = 2
   url_tag = "scrapeUrl"
@@ -578,7 +579,7 @@
   urls = $AZMON_DS_PROM_URLS
 
   fieldpass = $AZMON_DS_PROM_FIELDPASS
-  
+
   fielddrop = $AZMON_DS_PROM_FIELDDROP
 
   metric_version = 2

installer/datafiles/base_container.data

@@ -45,6 +45,8 @@ MAINTAINER:              'Microsoft Corporation'
 /opt/microsoft/omsagent/plugin/DockerApiClient.rb;												source/code/plugin/DockerApiClient.rb;				644; root; root
 /opt/microsoft/omsagent/plugin/DockerApiRestHelper.rb;											source/code/plugin/DockerApiRestHelper.rb;			644; root; root
 /opt/microsoft/omsagent/plugin/in_containerinventory.rb;										source/code/plugin/in_containerinventory.rb;		644; root; root
+/opt/microsoft/omsagent/plugin/kubernetes_container_inventory.rb;							source/code/plugin/kubernetes_container_inventory.rb;		644; root; root
+
 
 /opt/microsoft/omsagent/plugin/out_mdm.rb;                                                  source/code/plugin/out_mdm.rb; 644; root; root
 /opt/microsoft/omsagent/plugin/filter_cadvisor2mdm.rb;                                      source/code/plugin/filter_cadvisor2mdm.rb; 644; root; root
@@ -106,6 +108,7 @@ MAINTAINER:              'Microsoft Corporation'
 /opt/td-agent-bit/bin/out_oms.so;                                intermediate/${{BUILD_CONFIGURATION}}/out_oms.so;  	755; root; root
 /etc/opt/microsoft/docker-cimprov/td-agent-bit.conf;			    installer/conf/td-agent-bit.conf;                    644; root; root
 /etc/opt/microsoft/docker-cimprov/td-agent-bit-rs.conf;			    installer/conf/td-agent-bit-rs.conf;                    644; root; root
+/etc/opt/microsoft/docker-cimprov/azm-containers-parser.conf;	    installer/conf/azm-containers-parser.conf;                    644; root; root
 /etc/opt/microsoft/docker-cimprov/out_oms.conf;			            installer/conf/out_oms.conf;                    644; root; root
 /etc/opt/microsoft/docker-cimprov/telegraf.conf;			        installer/conf/telegraf.conf;                    644; root; root
 /etc/opt/microsoft/docker-cimprov/telegraf-rs.conf;			        installer/conf/telegraf-rs.conf;                    644; root; root

source/code/go/src/plugins/glide.yaml

@@ -3,6 +3,8 @@ import:
 - package: github.com/fluent/fluent-bit-go
   subpackages:
   - output
+- package: github.com/google/uuid
+  version: ^1.1.0
 - package: gopkg.in/natefinch/lumberjack.v2
   version: ^2.1.0
 - package: k8s.io/apimachinery

source/code/go/src/plugins/oms.go

@@ -14,6 +14,7 @@ import (
 	"time"
 
 	"github.com/fluent/fluent-bit-go/output"
+	"github.com/google/uuid"
 
 	lumberjack "gopkg.in/natefinch/lumberjack.v2"
 
@@ -37,6 +38,9 @@ const ResourceIdEnv = "AKS_RESOURCE_ID"
 //env variable which has ResourceName for NON-AKS
 const ResourceNameEnv = "ACS_RESOURCE_NAME"
 
+//env variable which has container run time name
+const ContainerRuntimeEnv = "CONTAINER_RUNTIME"
+
 // Origin prefix for telegraf Metrics (used as prefix for origin field & prefix for azure monitor specific tags and also for custom-metrics telemetry )
 const TelegrafMetricOriginPrefix = "container.azm.ms"
 
@@ -93,7 +97,9 @@ var (
 	//KubeMonAgentEvents skip first flush
 	skipKubeMonEventsFlush bool
 	// enrich container logs (when true this will add the fields - timeofcommand, containername & containerimage)
-	enrichContainerLogs bool
+	enrichContainerLogs bool		
+	// container runtime engine configured on the kubelet
+	containerRuntime string
 )
 
 var (
@@ -133,6 +139,12 @@ var (
 	Log = FLBLogger.Printf
 )
 
+var (
+	dockerCimprovVersion = "9.0.0.0"
+	agentName = "ContainerAgent"
+	userAgent = ""
+)
+
 // DataItem represents the object corresponding to the json that is sent by fluentbit tail plugin
 type DataItem struct {
 	LogEntry              string `json:"LogEntry"`
@@ -513,6 +525,9 @@ func flushKubeMonAgentEventRecords() {
 				} else {
 					req, _ := http.NewRequest("POST", OMSEndpoint, bytes.NewBuffer(marshalled))
 					req.Header.Set("Content-Type", "application/json")
+					req.Header.Set("User-Agent", userAgent )
+					reqId := uuid.New().String()
+					req.Header.Set("X-Request-ID", reqId)
 					//expensive to do string len for every request, so use a flag
 					if ResourceCentric == true {
 						req.Header.Set("x-ms-AzureResourceId", ResourceID)
@@ -527,7 +542,7 @@ func flushKubeMonAgentEventRecords() {
 						Log("Failed to flush %d records after %s", len(laKubeMonAgentEventsRecords), elapsed)
 					} else if resp == nil || resp.StatusCode != 200 {
 						if resp != nil {
-							Log("Status %s Status Code %d", resp.Status, resp.StatusCode)
+							Log(" RequestId %s Status %s Status Code %d", reqId, resp.Status, resp.StatusCode)
 						}
 						Log("Failed to flush %d records after %s", len(laKubeMonAgentEventsRecords), elapsed)
 					} else {
@@ -656,6 +671,9 @@ func PostTelegrafMetricsToLA(telegrafRecords []map[interface{}]interface{}) int
 
 	//set headers
 	req.Header.Set("x-ms-date", time.Now().Format(time.RFC3339))
+	req.Header.Set("User-Agent", userAgent )
+	reqId := uuid.New().String()
+	req.Header.Set("X-Request-ID", reqId)
 
 	//expensive to do string len for every request, so use a flag
 	if ResourceCentric == true {
@@ -669,31 +687,34 @@ func PostTelegrafMetricsToLA(telegrafRecords []map[interface{}]interface{}) int
 	if err != nil {
 		message := fmt.Sprintf("PostTelegrafMetricsToLA::Error:(retriable) when sending %v metrics. duration:%v err:%q \n", len(laMetrics), elapsed, err.Error())
 		Log(message)
-		UpdateNumTelegrafMetricsSentTelemetry(0, 1)
+		UpdateNumTelegrafMetricsSentTelemetry(0, 1, 0)
 		return output.FLB_RETRY
 	}
 
 	if resp == nil || resp.StatusCode != 200 {
 		if resp != nil {
-			Log("PostTelegrafMetricsToLA::Error:(retriable) Response Status %v Status Code %v", resp.Status, resp.StatusCode)
+			Log("PostTelegrafMetricsToLA::Error:(retriable) RequestID %s Response Status %v Status Code %v", reqId, resp.Status, resp.StatusCode)
+		}
+		if resp != nil && resp.StatusCode == 429 {
+			UpdateNumTelegrafMetricsSentTelemetry(0, 1, 1)
 		}
-		UpdateNumTelegrafMetricsSentTelemetry(0, 1)
 		return output.FLB_RETRY
 	}
 
 	defer resp.Body.Close()
 
 	numMetrics := len(laMetrics)
-	UpdateNumTelegrafMetricsSentTelemetry(numMetrics, 0)
+	UpdateNumTelegrafMetricsSentTelemetry(numMetrics, 0, 0)
 	Log("PostTelegrafMetricsToLA::Info:Successfully flushed %v records in %v", numMetrics, elapsed)
 
 	return output.FLB_OK
 }
 
-func UpdateNumTelegrafMetricsSentTelemetry(numMetricsSent int, numSendErrors int) {
+func UpdateNumTelegrafMetricsSentTelemetry(numMetricsSent int, numSendErrors int, numSend429Errors int) {
 	ContainerLogTelemetryMutex.Lock()
 	TelegrafMetricsSentCount += float64(numMetricsSent)
 	TelegrafMetricsSendErrorCount += float64(numSendErrors)
+	TelegrafMetricsSend429ErrorCount += float64(numSend429Errors)
 	ContainerLogTelemetryMutex.Unlock()
 }
 
@@ -734,9 +755,11 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int {
 
 		stringMap := make(map[string]string)
 
-		stringMap["LogEntry"] = ToString(record["log"])
+		logEntry := ToString(record["log"])
+		logEntryTimeStamp := ToString(record["time"])			
+		stringMap["LogEntry"] = logEntry
 		stringMap["LogEntrySource"] = logEntrySource
-		stringMap["LogEntryTimeStamp"] = ToString(record["time"])
+		stringMap["LogEntryTimeStamp"] = logEntryTimeStamp
 		stringMap["SourceSystem"] = "Containers"
 		stringMap["Id"] = containerID
 
@@ -781,7 +804,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int {
 		if dataItem.LogEntryTimeStamp != "" {
 			loggedTime, e := time.Parse(time.RFC3339, dataItem.LogEntryTimeStamp)
 			if e != nil {
-				message := fmt.Sprintf("Error while converting LogEntryTimeStamp for telemetry purposes: %s", e.Error())
+				message := fmt.Sprintf("containerId: %s Error while converting LogEntryTimeStamp for telemetry purposes: %s", dataItem.ID, e.Error())
 				Log(message)
 				SendException(message)
 			} else {
@@ -810,6 +833,9 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int {
 
 		req, _ := http.NewRequest("POST", OMSEndpoint, bytes.NewBuffer(marshalled))
 		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("User-Agent", userAgent )
+		reqId := uuid.New().String()
+		req.Header.Set("X-Request-ID", reqId)
 		//expensive to do string len for every request, so use a flag
 		if ResourceCentric == true {
 			req.Header.Set("x-ms-AzureResourceId", ResourceID)
@@ -830,7 +856,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int {
 
 		if resp == nil || resp.StatusCode != 200 {
 			if resp != nil {
-				Log("Status %s Status Code %d", resp.Status, resp.StatusCode)
+				Log("RequestId %s Status %s Status Code %d", reqId, resp.Status, resp.StatusCode)
 			}
 			return output.FLB_RETRY
 		}
@@ -958,6 +984,21 @@ func InitializePlugin(pluginConfPath string, agentVersion string) {
 		Log("ResourceID=%s", ResourceID)
 		Log("ResourceName=%s", ResourceName)
 	}
+
+	// log runtime info for debug purpose
+	containerRuntime = os.Getenv(ContainerRuntimeEnv)		
+	Log("Container Runtime engine %s", containerRuntime)	
+
+
+	//set useragent to be used by ingestion 
+	docker_cimprov_version := strings.TrimSpace(os.Getenv("DOCKER_CIMPROV_VERSION"))
+	if len(docker_cimprov_version) > 0 {
+		dockerCimprovVersion = docker_cimprov_version
+	}
+
+	userAgent = fmt.Sprintf("%s/%s", agentName, dockerCimprovVersion)
+
+	Log("Usage-Agent = %s \n", userAgent)
 
 	// Initialize image,name map refresh ticker
 	containerInventoryRefreshInterval, err := strconv.Atoi(pluginConfig["container_inventory_refresh_interval"])

source/code/go/src/plugins/telemetry.go

@@ -34,6 +34,8 @@ var (
 	TelegrafMetricsSentCount float64
 	//Tracks the number of send errors between telemetry ticker periods (uses ContainerLogTelemetryTicker)
 	TelegrafMetricsSendErrorCount float64
+    //Tracks the number of 429 (throttle) errors between telemetry ticker periods (uses ContainerLogTelemetryTicker)
+	TelegrafMetricsSend429ErrorCount float64
 )
 
 const (
@@ -49,6 +51,7 @@ const (
 	metricNameAgentLogProcessingMaxLatencyMs          = "ContainerLogsAgentSideLatencyMs"
 	metricNameNumberofTelegrafMetricsSentSuccessfully = "TelegrafMetricsSentCount"
 	metricNameNumberofSendErrorsTelegrafMetrics       = "TelegrafMetricsSendErrorCount"
+	metricNameNumberofSend429ErrorsTelegrafMetrics    = "TelegrafMetricsSend429ErrorCount"
 
 	defaultTelemetryPushIntervalSeconds = 300
 
@@ -78,8 +81,10 @@ func SendContainerLogPluginMetrics(telemetryPushIntervalProperty string) {
 		logSizeRate := FlushedRecordsSize / float64(elapsed/time.Second)
 		telegrafMetricsSentCount := TelegrafMetricsSentCount
 		telegrafMetricsSendErrorCount := TelegrafMetricsSendErrorCount
+		telegrafMetricsSend429ErrorCount := TelegrafMetricsSend429ErrorCount
 		TelegrafMetricsSentCount = 0.0
 		TelegrafMetricsSendErrorCount = 0.0
+		TelegrafMetricsSend429ErrorCount = 0.0
 		FlushedRecordsCount = 0.0
 		FlushedRecordsSize = 0.0
 		FlushedRecordsTimeTaken = 0.0
@@ -103,7 +108,12 @@ func SendContainerLogPluginMetrics(telemetryPushIntervalProperty string) {
 			TelemetryClient.Track(logLatencyMetric)
 		}
 		TelemetryClient.Track(appinsights.NewMetricTelemetry(metricNameNumberofTelegrafMetricsSentSuccessfully, telegrafMetricsSentCount))
-		TelemetryClient.Track(appinsights.NewMetricTelemetry(metricNameNumberofSendErrorsTelegrafMetrics, telegrafMetricsSendErrorCount))
+		if telegrafMetricsSendErrorCount > 0.0 {
+			TelemetryClient.Track(appinsights.NewMetricTelemetry(metricNameNumberofSendErrorsTelegrafMetrics, telegrafMetricsSendErrorCount))
+		}
+		if telegrafMetricsSend429ErrorCount > 0.0 {
+			TelemetryClient.Track(appinsights.NewMetricTelemetry(metricNameNumberofSend429ErrorsTelegrafMetrics, telegrafMetricsSend429ErrorCount))
+		}
 		start = time.Now()
 	}
 }