Merge ci_feature into ci_feature_prod

README.md

@@ -11,6 +11,22 @@ additional questions or comments.
 
 Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate the agent build dates (not release dates)
 
+### 01/20/2019 - Version microsoft/oms:ciprod02202019
+- Container logs enrichment optimization
+  * Get container meta data only for containers in current node (vs cluster before)
+- Update fluent bit 0.13.7 => 0.14.4
+  * This fixes the escaping issue in the container logs
+- Mooncake cloud support for agent (AKS only)
+  * Ability to disable agent telemetry
+  * Ability to onboard and ingest to mooncake cloud
+- Add & populate 'ContainerStatusReason'  column to KubePodInventory
+- Alertable (custom) metrics (to AzureMonitor - only for AKS clusters)
+  * Cpuusagenanocores & % metric
+  * MemoryWorkingsetBytes & % metric
+  * MemoryRssBytes & % metric
+  * Podcount by node, phase & namespace metric
+  * Nodecount metric
+
 ### 01/09/2018 - Version microsoft/oms:ciprod01092019
 - Omsagent - 1.8.1.256 (nov 2018 release)
 - Persist fluentbit state between container restarts
@@ -25,7 +41,7 @@ Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate t
 - Agent telemetry - ContainerLogsAgentSideLatencyMs
 - Agent telemetry - PodCount
 - Agent telemetry - ControllerCount
--	Agent telemetry - K8S Version
+- Agent telemetry - K8S Version
 - Agent telemetry - NodeCoreCapacity
 - Agent telemetry - NodeMemoryCapacity
 - Agent telemetry - KubeEvents (exceptions)

installer/conf/container.conf

@@ -23,6 +23,14 @@
   log_level debug
 </source>
 
+#custom_metrics_mdm filter plugin
+<filter mdm.cadvisorperf**>
+  type filter_cadvisor2mdm
+  custom_metrics_azure_regions eastus,southcentralus,westcentralus,westus2,southeastasia,northeurope,westEurope
+  metrics_to_collect cpuUsageNanoCores,memoryWorkingSetBytes,memoryRssBytes
+  log_level info
+</filter>
+
 <match oms.containerinsights.containerinventory**>
   type out_oms
   log_level debug
@@ -52,3 +60,19 @@
   retry_wait 30s
   max_retry_wait 9m
 </match>
+
+<match mdm.cadvisorperf**>
+  type out_mdm
+  log_level debug
+  num_threads 5
+  buffer_chunk_limit 20m
+  buffer_type file
+  buffer_path %STATE_DIR_WS%/out_mdm_cdvisorperf*.buffer
+  buffer_queue_limit 20
+  buffer_queue_full_action drop_oldest_chunk
+  flush_interval 20s
+  retry_limit 10
+  retry_wait 30s
+  max_retry_wait 9m
+  retry_mdm_post_wait_minutes 60
+</match>

installer/conf/kube.conf

@@ -47,6 +47,12 @@
   log_level debug
 </source>
 
+<filter mdm.kubepodinventory** mdm.kubenodeinventory**>
+  type filter_inventory2mdm
+  custom_metrics_azure_regions eastus,southcentralus,westcentralus,westus2,southeastasia,northeurope,westEurope
+  log_level info
+</filter>
+
 <match oms.containerinsights.KubePodInventory**>
   type out_oms
   log_level debug
@@ -119,8 +125,8 @@
   max_retry_wait 9m
 </match>
 
-<match oms.api.ContainerNodeInventory**>
-  type out_oms_api
+<match oms.containerinsights.ContainerNodeInventory**>
+  type out_oms
   log_level debug
   buffer_chunk_limit 20m
   buffer_type file
@@ -146,3 +152,19 @@
   retry_wait 30s
   max_retry_wait 9m
 </match>
+
+<match mdm.kubepodinventory** mdm.kubenodeinventory** >
+  type out_mdm
+  log_level debug
+  num_threads 5
+  buffer_chunk_limit 20m
+  buffer_type file
+  buffer_path /var/opt/microsoft/omsagent/6bb1e963-b08c-43a8-b708-1628305e964a/state/out_mdm_*.buffer
+  buffer_queue_limit 20
+  buffer_queue_full_action drop_oldest_chunk
+  flush_interval 20s
+  retry_limit 10
+  retry_wait 30s
+  max_retry_wait 9m
+  retry_mdm_post_wait_minutes 60
+</match>

installer/conf/td-agent-bit.conf

@@ -28,5 +28,5 @@
     EnableTelemetry                 true
     TelemetryPushIntervalSeconds    300
     Match                           oms.container.log.*
-    AgentVersion                    ciprod01092019
+    AgentVersion                    ciprod02202019
 

installer/datafiles/base_container.data

@@ -36,13 +36,19 @@ MAINTAINER:              'Microsoft Corporation'
 /opt/microsoft/omsagent/plugin/in_cadvisor_perf.rb;			        source/code/plugin/in_cadvisor_perf.rb;			    	644; root; root
 /opt/microsoft/omsagent/plugin/in_kube_services.rb;			        source/code/plugin/in_kube_services.rb;			    	644; root; root
 /opt/microsoft/omsagent/plugin/in_kube_nodes.rb;			        source/code/plugin/in_kube_nodes.rb;			    	644; root; root
+/opt/microsoft/omsagent/plugin/filter_inventory2mdm.rb;			     source/code/plugin/filter_inventory2mdm.rb;			    	644; root; root
+/opt/microsoft/omsagent/plugin/CustomMetricsUtils.rb;			     source/code/plugin/CustomMetricsUtils.rb;			    	644; root; root
+
 
 /opt/microsoft/omsagent/plugin/ApplicationInsightsUtility.rb;									source/code/plugin/ApplicationInsightsUtility.rb;	644; root; root
 /opt/microsoft/omsagent/plugin/ContainerInventoryState.rb;										source/code/plugin/ContainerInventoryState.rb;		644; root; root
 /opt/microsoft/omsagent/plugin/DockerApiClient.rb;												source/code/plugin/DockerApiClient.rb;				644; root; root
 /opt/microsoft/omsagent/plugin/DockerApiRestHelper.rb;											source/code/plugin/DockerApiRestHelper.rb;			644; root; root
 /opt/microsoft/omsagent/plugin/in_containerinventory.rb;										source/code/plugin/in_containerinventory.rb;		644; root; root
 
+/opt/microsoft/omsagent/plugin/out_mdm.rb;                                                  source/code/plugin/out_mdm.rb; 644; root; root
+/opt/microsoft/omsagent/plugin/filter_cadvisor2mdm.rb;                                      source/code/plugin/filter_cadvisor2mdm.rb; 644; root; root
+
 /opt/microsoft/omsagent/plugin/lib/application_insights/version.rb;								source/code/plugin/lib/application_insights/version.rb; 644; root; root
 /opt/microsoft/omsagent/plugin/lib/application_insights/rack/track_request.rb;					source/code/plugin/lib/application_insights/rack/track_request.rb;      644; root; root
 /opt/microsoft/omsagent/plugin/lib/application_insights/unhandled_exception.rb;					source/code/plugin/lib/application_insights/unhandled_exception.rb;     644; root; root
@@ -170,6 +176,14 @@ touch /var/opt/microsoft/docker-cimprov/log/kubernetes_perf_log.txt
 chmod 666 /var/opt/microsoft/docker-cimprov/log/kubernetes_perf_log.txt
 chown omsagent:omiusers /var/opt/microsoft/docker-cimprov/log/kubernetes_perf_log.txt
 
+touch /var/opt/microsoft/docker-cimprov/log/filter_cadvisor2mdm.log
+chmod 666 /var/opt/microsoft/docker-cimprov/log/filter_cadvisor2mdm.log
+chown omsagent:omiusers /var/opt/microsoft/docker-cimprov/log/filter_cadvisor2mdm.log
+
+touch /var/opt/microsoft/docker-cimprov/log/filter_inventory2mdm.log
+chmod 666 /var/opt/microsoft/docker-cimprov/log/filter_inventory2mdm.log
+chown omsagent:omiusers /var/opt/microsoft/docker-cimprov/log/filter_inventory2mdm.log
+
 mv /etc/opt/microsoft/docker-cimprov/container.conf /etc/opt/microsoft/omsagent/sysconf/omsagent.d/container.conf
 chown omsagent:omsagent /etc/opt/microsoft/omsagent/sysconf/omsagent.d/container.conf
 

source/code/go/src/plugins/oms.go

@@ -77,15 +77,15 @@ var (
 
 // DataItem represents the object corresponding to the json that is sent by fluentbit tail plugin
 type DataItem struct {
-	LogEntry          		string `json:"LogEntry"`
-	LogEntrySource    		string `json:"LogEntrySource"`
-	LogEntryTimeStamp 		string `json:"LogEntryTimeStamp"`
-	LogEntryTimeOfCommand	string `json:"TimeOfCommand"`
-	ID                string `json:"Id"`
-	Image             string `json:"Image"`
-	Name              string `json:"Name"`
-	SourceSystem      string `json:"SourceSystem"`
-	Computer          string `json:"Computer"`
+	LogEntry              string `json:"LogEntry"`
+	LogEntrySource        string `json:"LogEntrySource"`
+	LogEntryTimeStamp     string `json:"LogEntryTimeStamp"`
+	LogEntryTimeOfCommand string `json:"TimeOfCommand"`
+	ID                    string `json:"Id"`
+	Image                 string `json:"Image"`
+	Name                  string `json:"Name"`
+	SourceSystem          string `json:"SourceSystem"`
+	Computer              string `json:"Computer"`
 }
 
 // ContainerLogBlob represents the object corresponding to the payload that is sent to the ODS end point
@@ -137,7 +137,10 @@ func updateContainerImageNameMaps() {
 		_imageIDMap := make(map[string]string)
 		_nameIDMap := make(map[string]string)
 
-		pods, err := ClientSet.CoreV1().Pods("").List(metav1.ListOptions{})
+		listOptions := metav1.ListOptions{}
+		listOptions.FieldSelector = fmt.Sprintf("spec.nodeName=%s", Computer)
+		pods, err := ClientSet.CoreV1().Pods("").List(listOptions)
+
 		if err != nil {
 			message := fmt.Sprintf("Error getting pods %s\nIt is ok to log here and continue, because the logs will be missing image and Name, but the logs will still have the containerID", err.Error())
 			Log(message)
@@ -244,31 +247,31 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int {
 		if val, ok := imageIDMap[containerID]; ok {
 			stringMap["Image"] = val
 		} else {
-			Log("ContainerId %s not present in Map ", containerID)
+			Log("ContainerId %s not present in Name Map ", containerID)
 		}
 
 		if val, ok := nameIDMap[containerID]; ok {
 			stringMap["Name"] = val
 		} else {
-			Log("ContainerId %s not present in Map ", containerID)
+			Log("ContainerId %s not present in Image Map ", containerID)
 		}
 
 
 		dataItem := DataItem{
-			ID:                		stringMap["Id"],
-			LogEntry:          		stringMap["LogEntry"],
-			LogEntrySource:    		stringMap["LogEntrySource"],
-			LogEntryTimeStamp: 		stringMap["LogEntryTimeStamp"],
-			LogEntryTimeOfCommand: 	start.Format(time.RFC3339),
-			SourceSystem:      		stringMap["SourceSystem"],
-			Computer:          		Computer,
-			Image:             		stringMap["Image"],
-			Name:              		stringMap["Name"],
+			ID:                    stringMap["Id"],
+			LogEntry:              stringMap["LogEntry"],
+			LogEntrySource:        stringMap["LogEntrySource"],
+			LogEntryTimeStamp:     stringMap["LogEntryTimeStamp"],
+			LogEntryTimeOfCommand: start.Format(time.RFC3339),
+			SourceSystem:          stringMap["SourceSystem"],
+			Computer:              Computer,
+			Image:                 stringMap["Image"],
+			Name:                  stringMap["Name"],
 		}
 
 		dataItems = append(dataItems, dataItem)
 		loggedTime, e := time.Parse(time.RFC3339, dataItem.LogEntryTimeStamp)
-		if e!= nil {
+		if e != nil {
 			message := fmt.Sprintf("Error while converting LogEntryTimeStamp for telemetry purposes: %s", e.Error())
 			Log(message)
 			SendException(message)

source/code/go/src/plugins/telemetry.go

@@ -120,6 +120,11 @@ func InitializeTelemetryClient(agentVersion string) (int, error) {
 	}
 
 	TelemetryClient = appinsights.NewTelemetryClient(string(decIkey))
+	telemetryOffSwitch := os.Getenv("DISABLE_TELEMETRY")
+	if strings.Compare(strings.ToLower(telemetryOffSwitch), "true") == 0 {
+		Log("Appinsights telemetry is disabled \n")
+		TelemetryClient.SetIsEnabled(false)
+	}
 
 	CommonProperties = make(map[string]string)
 	CommonProperties["Computer"] = Computer

source/code/plugin/ApplicationInsightsUtility.rb

@@ -61,9 +61,16 @@ def initializeUtility()
                 @@CustomProperties['AgentVersion'] = ENV[@@EnvAgentVersion]
                 @@CustomProperties['ControllerType'] = ENV[@@EnvControllerType]
                 encodedAppInsightsKey = ENV[@@EnvApplicationInsightsKey]
-                if !encodedAppInsightsKey.nil?
+
+                #Check if telemetry is turned off
+                telemetryOffSwitch = ENV['DISABLE_TELEMETRY']
+                if telemetryOffSwitch && !telemetryOffSwitch.nil? && !telemetryOffSwitch.empty? && telemetryOffSwitch.downcase == "true".downcase
+                    $log.warn("AppInsightsUtility: Telemetry is disabled")
+                    @@Tc = ApplicationInsights::TelemetryClient.new
+                elsif !encodedAppInsightsKey.nil?
                     decodedAppInsightsKey = Base64.decode64(encodedAppInsightsKey)
                     @@Tc = ApplicationInsights::TelemetryClient.new decodedAppInsightsKey
+
                 end
             rescue => errorStr
                 $log.warn("Exception in AppInsightsUtility: initilizeUtility - error: #{errorStr}")
@@ -91,7 +98,7 @@ def sendHeartBeatEvent(pluginName)
             end
         end
 
-        def sendCustomMetric(pluginName, properties)
+        def sendLastProcessedContainerInventoryCountMetric(pluginName, properties)
             begin
                 if !(@@Tc.nil?)
                     @@Tc.track_metric 'LastProcessedContainerInventoryCount', properties['ContainerCount'], 
@@ -105,6 +112,21 @@ def sendCustomMetric(pluginName, properties)
             end
         end
 
+        def sendCustomEvent(eventName, properties)
+            begin
+                if @@CustomProperties.empty? || @@CustomProperties.nil?
+                    initializeUtility()
+                end 
+                if !(@@Tc.nil?)
+                    @@Tc.track_event eventName, :properties => @@CustomProperties
+                    @@Tc.flush
+                    $log.info("AppInsights Custom Event #{eventName} sent successfully")
+                end
+            rescue => errorStr
+                $log.warn("Exception in AppInsightsUtility: sendCustomEvent - error: #{errorStr}")
+            end
+        end
+
         def sendExceptionTelemetry(errorStr)
             begin
                 if @@CustomProperties.empty? || @@CustomProperties.nil?
@@ -132,7 +154,7 @@ def sendTelemetry(pluginName, properties)
                 end
                 @@CustomProperties['Computer'] = properties['Computer']
                 sendHeartBeatEvent(pluginName)
-                sendCustomMetric(pluginName, properties)
+                sendLastProcessedContainerInventoryCountMetric(pluginName, properties)
             rescue => errorStr
                 $log.warn("Exception in AppInsightsUtility: sendTelemetry - error: #{errorStr}")
             end

source/code/plugin/CustomMetricsUtils.rb

@@ -0,0 +1,26 @@
+#!/usr/local/bin/ruby
+# frozen_string_literal: true
+
+class CustomMetricsUtils
+    def initialize
+    end
+
+    class << self
+        def check_custom_metrics_availability(custom_metric_regions)
+            aks_region = ENV['AKS_REGION']
+            aks_resource_id = ENV['AKS_RESOURCE_ID']
+            if aks_region.to_s.empty? && aks_resource_id.to_s.empty?
+                false # This will also take care of AKS-Engine Scenario. AKS_REGION/AKS_RESOURCE_ID is not set for AKS-Engine. Only ACS_RESOURCE_NAME is set
+            end
+
+            custom_metrics_regions_arr = custom_metric_regions.split(',')
+            custom_metrics_regions_hash = custom_metrics_regions_arr.map {|x| [x.downcase,true]}.to_h
+
+            if custom_metrics_regions_hash.key?(aks_region.downcase)
+                true
+            else 
+                false
+            end
+        end
+    end
+end