Gangams/jan 2022 release tasks 2

build/linux/installer/conf/telegraf-prom-side-car.conf

@@ -111,6 +111,26 @@
   data_format = "json"
   namedrop = ["agent_telemetry", "file"]
 
+# [[outputs.application_insights]]
+#   ## Instrumentation key of the Application Insights resource.
+#   instrumentation_key = "$TELEMETRY_APPLICATIONINSIGHTS_KEY"
+
+#   ## Timeout for closing (default: 5s).
+#   # timeout = "5s"
+
+#   ## Enable additional diagnostic logging.
+#   enable_diagnostic_logging = false
+
+#   ## Context Tag Sources add Application Insights context tags to a tag value.
+#   ##
+#   ## For list of allowed context tag keys see:
+#   ## https://github.com/Microsoft/ApplicationInsights-Go/blob/master/appinsights/contracts/contexttagkeys.go
+#   # [outputs.application_insights.context_tag_sources]
+#   #   "ai.cloud.role" = "kubernetes_container_name"
+#   #   "ai.cloud.roleInstance" = "kubernetes_pod_name"
+#   namepass = ["t.azm.ms/agent_telemetry"]
+  #tagdrop = ["nodeName"]
+
 ###############################################################################
 #                            PROCESSOR PLUGINS                                #
 ###############################################################################
@@ -119,9 +139,23 @@
   [processors.converter.fields]
     float = ["*"]
 
+###############################################################################
+#                            AGGREGATOR PLUGINS                               #
+###############################################################################
+# [[aggregators.quantile]]
+#   period = "30m"
+#   drop_original = true
+#   quantiles = [0.95]
+#   algorithm = "t-digest"
+#   compression = 100.0
+#   namepass = ["agent_telemetry"]
+
+###############################################################################
+#                            INPUT PLUGINS                                    #
+###############################################################################
 # Dummy plugin to test out toml parsing happens properly
 [[inputs.file]]
-  interval = "24h" 
+  interval = "24h"
   files = ["test.json"]
   data_format = "json"
 
@@ -166,3 +200,20 @@ $AZMON_TELEGRAF_CUSTOM_PROM_PLUGINS_WITH_NAMESPACE_FILTER
 
 ## OSM Prometheus configuration
 $AZMON_TELEGRAF_OSM_PROM_PLUGINS
+
+# [[inputs.procstat]]
+#  name_prefix="t.azm.ms/"
+#  exe = "mdsd"
+#  interval = "60s"
+#  pid_finder = "native"
+#  pid_tag = true
+#  name_override = "agent_telemetry"
+#  fieldpass = ["cpu_usage", "memory_rss"]
+#  [inputs.procstat.tags]
+#    Computer = "$NODE_NAME"
+#    AgentVersion = "$AGENT_VERSION"
+#    ControllerType = "$CONTROLLER_TYPE"
+#    AKS_RESOURCE_ID = "$TELEMETRY_AKS_RESOURCE_ID"
+#    ACSResourceName = "$TELEMETRY_ACS_RESOURCE_NAME"
+#    Region = "$TELEMETRY_AKS_REGION"
+#    ContainerType = "$CONTAINER_TYPE"

build/linux/installer/conf/telegraf-rs.conf

@@ -124,6 +124,26 @@
   namedrop = ["agent_telemetry", "file"]
   #tagdrop = ["AgentVersion","AKS_RESOURCE_ID", "ACS_RESOURCE_NAME", "Region","ClusterName","ClusterType", "Computer", "ControllerType"]
 
+# [[outputs.application_insights]]
+#   ## Instrumentation key of the Application Insights resource.
+#   instrumentation_key = "$TELEMETRY_APPLICATIONINSIGHTS_KEY"
+
+#   ## Timeout for closing (default: 5s).
+#   # timeout = "5s"
+
+#   ## Enable additional diagnostic logging.
+#   enable_diagnostic_logging = false
+
+#   ## Context Tag Sources add Application Insights context tags to a tag value.
+#   ##
+#   ## For list of allowed context tag keys see:
+#   ## https://github.com/Microsoft/ApplicationInsights-Go/blob/master/appinsights/contracts/contexttagkeys.go
+#   # [outputs.application_insights.context_tag_sources]
+#   #   "ai.cloud.role" = "kubernetes_container_name"
+#   #   "ai.cloud.roleInstance" = "kubernetes_pod_name"
+#   namepass = ["t.azm.ms/agent_telemetry"]
+  #tagdrop = ["nodeName"]
+
 ###############################################################################
 #                            PROCESSOR PLUGINS                                #
 ###############################################################################
@@ -293,6 +313,13 @@
 ###############################################################################
 #                            AGGREGATOR PLUGINS                               #
 ###############################################################################
+# [[aggregators.quantile]]
+#   period = "30m"
+#   drop_original = true
+#   quantiles = [0.95]
+#   algorithm = "t-digest"
+#   compression = 100.0
+#   namepass = ["agent_telemetry"]
 
 # # Keep the aggregate basicstats of each metric passing through.
 # [[aggregators.basicstats]]
@@ -369,15 +396,15 @@
 #  report_active = true
 #  fieldpass = ["usage_active","cluster","node","host","device"]
 #  taginclude = ["cluster","cpu","node"]
-  
+
 
 
 # Read metrics about disk usage by mount point
 #[[inputs.disk]]
   ## By default stats will be gathered for all mount points.
   ## Set mount_points will restrict the stats to only the specified mount points.
   # mount_points = ["/"]
-  
+
   ## Ignore mount points by filesystem type.
 #  ignore_fs = ["tmpfs", "devtmpfs", "devfs", "overlay", "aufs", "squashfs"]
 #  fieldpass = ["free", "used", "used_percent"]
@@ -520,7 +547,7 @@
 
 # Dummy plugin to test out toml parsing happens properly
 [[inputs.file]]
-  interval = "24h" 
+  interval = "24h"
   files = ["test.json"]
   data_format = "json"
 
@@ -530,10 +557,10 @@
 
   ## An array of urls to scrape metrics from.
   urls = $AZMON_TELEGRAF_CUSTOM_PROM_URLS
-  
+
   ## An array of Kubernetes services to scrape metrics from.
   kubernetes_services = $AZMON_TELEGRAF_CUSTOM_PROM_K8S_SERVICES
-  
+
   ## Scrape Kubernetes pods for the following prometheus annotations:
   ## - prometheus.io/scrape: Enable scraping for this pod
   ## - prometheus.io/scheme: If the metrics endpoint is secured then you will need to
@@ -648,3 +675,18 @@ $AZMON_TELEGRAF_OSM_PROM_PLUGINS
   #[inputs.prometheus.tagpass]
   #  operation_type = ["create_container", "remove_container", "pull_image"]
 
+# [[inputs.procstat]]
+#  name_prefix="t.azm.ms/"
+#  exe = "mdsd"
+#  interval = "60s"
+#  pid_finder = "native"
+#  pid_tag = true
+#  name_override = "agent_telemetry"
+#  fieldpass = ["cpu_usage", "memory_rss"]
+#  [inputs.procstat.tags]
+#    Computer = "$NODE_NAME"
+#    AgentVersion = "$AGENT_VERSION"
+#    ControllerType = "$CONTROLLER_TYPE"
+#    AKS_RESOURCE_ID = "$TELEMETRY_AKS_RESOURCE_ID"
+#    ACSResourceName = "$TELEMETRY_ACS_RESOURCE_NAME"
+#    Region = "$TELEMETRY_AKS_REGION"

build/linux/installer/conf/telegraf.conf

@@ -158,6 +158,26 @@
   namepass = ["container.azm.ms/disk"]
   #fieldpass = ["used_percent"]
 
+# [[outputs.application_insights]]
+#   ## Instrumentation key of the Application Insights resource.
+#   instrumentation_key = "$TELEMETRY_APPLICATIONINSIGHTS_KEY"
+
+#   ## Timeout for closing (default: 5s).
+#   # timeout = "5s"
+
+#   ## Enable additional diagnostic logging.
+#   enable_diagnostic_logging = false
+
+  ## Context Tag Sources add Application Insights context tags to a tag value.
+  ##
+  ## For list of allowed context tag keys see:
+  ## https://github.com/Microsoft/ApplicationInsights-Go/blob/master/appinsights/contracts/contexttagkeys.go
+  # [outputs.application_insights.context_tag_sources]
+  #   "ai.cloud.role" = "kubernetes_container_name"
+  #   "ai.cloud.roleInstance" = "kubernetes_pod_name"
+  # namepass = ["agent_telemetry"]
+  #tagdrop = ["nodeName"]
+
 ###############################################################################
 #                            PROCESSOR PLUGINS                                #
 ###############################################################################
@@ -328,7 +348,13 @@
 ###############################################################################
 #                            AGGREGATOR PLUGINS                               #
 ###############################################################################
-
+# [[aggregators.quantile]]
+#   period = "30m"
+#   drop_original = true
+#   quantiles = [0.95]
+#   algorithm = "t-digest"
+#   compression = 100.0
+#   namepass = ["t.azm.ms/agent_telemetry"]
 # # Keep the aggregate basicstats of each metric passing through.
 # [[aggregators.basicstats]]
 #   ## General Aggregator Arguments:
@@ -407,7 +433,7 @@
 
 # Dummy plugin to test out toml parsing happens properly
 [[inputs.file]]
-  interval = "24h" 
+  interval = "24h"
   files = ["test.json"]
   data_format = "json"
 
@@ -550,14 +576,14 @@
   #fieldpass = ["numContainers", "numContainersRunning", "numContainersStopped", "numContainersPaused", "numContainerImages"]
 #  taginclude = ["nodeName"]
 
-#[[inputs.procstat]]
-#  #name_prefix="t.azm.ms/"
+# [[inputs.procstat]]
+#  name_prefix="t.azm.ms/"
 #  exe = "mdsd"
-#  interval = "10s"
+#  interval = "60s"
 #  pid_finder = "native"
 #  pid_tag = true
 #  name_override = "agent_telemetry"
-#  fieldpass = ["cpu_usage", "memory_rss", "memory_swap", "memory_vms", "memory_stack"]
+#  fieldpass = ["cpu_usage", "memory_rss"]
 #  [inputs.procstat.tags]
 #    Computer = "$NODE_NAME"
 #    AgentVersion = "$AGENT_VERSION"

build/linux/installer/datafiles/base_container.data

@@ -286,6 +286,8 @@ chmod 666 /var/opt/microsoft/docker-cimprov/log/fluent_forward_failed.log
 touch /var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log
 chmod 666 /var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log
 
+touch /var/opt/microsoft/docker-cimprov/log/appinsights_error.log
+chmod 666 /var/opt/microsoft/docker-cimprov/log/appinsights_error.log
 
 touch /var/opt/microsoft/docker-cimprov/log/fluentd.log
 chmod 666 /var/opt/microsoft/docker-cimprov/log/fluentd.log
@@ -309,7 +311,7 @@ if ${{PERFORMING_UPGRADE_NOT}}; then
    rmdir /etc/opt/microsoft/docker-cimprov/conf 2> /dev/null
    rmdir /etc/opt/microsoft/docker-cimprov 2> /dev/null
    rmdir /etc/opt/microsoft 2> /dev/null
-   rmdir /etc/opt 2> /dev/null  
+   rmdir /etc/opt 2> /dev/null
 fi
 
 %Preinstall_0

source/plugins/ruby/ApplicationInsightsUtility.rb

@@ -98,27 +98,34 @@ def initializeUtility()
         elsif !encodedAppInsightsKey.nil?
           decodedAppInsightsKey = Base64.decode64(encodedAppInsightsKey)
 
+          if @@isWindows
+            logPath = "/etc/omsagentwindows/appinsights_error.log"
+          else
+            logPath = "/var/opt/microsoft/docker-cimprov/log/appinsights_error.log"
+          end
+          aiLogger = Logger.new(logPath, 1, 2 * 1024 * 1024)
+
           #override ai endpoint if its available otherwise use default.
           if appInsightsEndpoint && !appInsightsEndpoint.nil? && !appInsightsEndpoint.empty?
             $log.info("AppInsightsUtility: Telemetry client uses overrided endpoint url : #{appInsightsEndpoint}")
             #telemetrySynchronousSender = ApplicationInsights::Channel::SynchronousSender.new appInsightsEndpoint
             #telemetrySynchronousQueue = ApplicationInsights::Channel::SynchronousQueue.new(telemetrySynchronousSender)
             #telemetryChannel = ApplicationInsights::Channel::TelemetryChannel.new nil, telemetrySynchronousQueue
             if !isProxyConfigured
-              sender = ApplicationInsights::Channel::AsynchronousSender.new appInsightsEndpoint
+              sender = ApplicationInsights::Channel::AsynchronousSender.new appInsightsEndpoint, aiLogger
             else
               $log.info("AppInsightsUtility: Telemetry client uses provided proxy configuration since proxy configured")
-              sender = ApplicationInsights::Channel::AsynchronousSender.new appInsightsEndpoint, @@proxy
+              sender = ApplicationInsights::Channel::AsynchronousSender.new appInsightsEndpoint, aiLogger, @@proxy
             end
             queue = ApplicationInsights::Channel::AsynchronousQueue.new sender
             channel = ApplicationInsights::Channel::TelemetryChannel.new nil, queue
             @@Tc = ApplicationInsights::TelemetryClient.new decodedAppInsightsKey, channel
           else
             if !isProxyConfigured
-              sender = ApplicationInsights::Channel::AsynchronousSender.new
+              sender = ApplicationInsights::Channel::AsynchronousSender.new nil, aiLogger
             else
               $log.info("AppInsightsUtility: Telemetry client uses provided proxy configuration since proxy configured")
-              sender = ApplicationInsights::Channel::AsynchronousSender.new nil, @@proxy
+              sender = ApplicationInsights::Channel::AsynchronousSender.new nil, aiLogger, @@proxy
             end
             queue = ApplicationInsights::Channel::AsynchronousQueue.new sender
             channel = ApplicationInsights::Channel::TelemetryChannel.new nil, queue

source/plugins/ruby/lib/application_insights/channel/asynchronous_sender.rb

@@ -1,5 +1,5 @@
-require_relative 'sender_base'
-require 'thread'
+require_relative "sender_base"
+require "thread"
 
 module ApplicationInsights
   module Channel
@@ -17,12 +17,13 @@ module Channel
     # If no queue items are found for {#send_time} seconds,  the worker thread
     # will shut down (and {#start} will need  to be called again).
     class AsynchronousSender < SenderBase
-      SERVICE_ENDPOINT_URI = 'https://dc.services.visualstudio.com/v2/track'
+      SERVICE_ENDPOINT_URI = "https://dc.services.visualstudio.com/v2/track"
       # Initializes a new instance of the class.
       # @param [String] service_endpoint_uri the address of the service to send
+      # @param [Logger] instance of the logger to write the logs (optional)
       # @param [Hash] proxy server configuration to send (optional)
       #   telemetry data to.
-      def initialize(service_endpoint_uri = SERVICE_ENDPOINT_URI, proxy = {})
+      def initialize(service_endpoint_uri = SERVICE_ENDPOINT_URI, logger = nil, proxy = {})
         # callers which requires proxy dont require to maintain service endpoint uri which potentially can change
         if service_endpoint_uri.nil? || service_endpoint_uri.empty?
           service_endpoint_uri = SERVICE_ENDPOINT_URI
@@ -33,7 +34,7 @@ def initialize(service_endpoint_uri = SERVICE_ENDPOINT_URI, proxy = {})
         @lock_work_thread = Mutex.new
         @work_thread = nil
         @start_notification_processed = true
-        super service_endpoint_uri, proxy
+        super service_endpoint_uri, logger, proxy
       end
 
       # The time span in seconds at which the the worker thread will check the
@@ -130,7 +131,7 @@ def run
         rescue Exception => e
           # Make sure work_thread sets to nil when it terminates abnormally
           @work_thread = nil
-          @logger.error('application_insights') { "Asynchronous sender work thread terminated abnormally: #{e.to_s}" }
+          @logger.error("application_insights") { "Asynchronous sender work thread terminated abnormally: #{e.to_s}" }
         end
       end
     end