diff --git a/build/linux/installer/conf/telegraf-prom-side-car.conf b/build/linux/installer/conf/telegraf-prom-side-car.conf index 1b6bab9f9..a94150fad 100644 --- a/build/linux/installer/conf/telegraf-prom-side-car.conf +++ b/build/linux/installer/conf/telegraf-prom-side-car.conf @@ -111,6 +111,26 @@ data_format = "json" namedrop = ["agent_telemetry", "file"] +# [[outputs.application_insights]] +# ## Instrumentation key of the Application Insights resource. +# instrumentation_key = "$TELEMETRY_APPLICATIONINSIGHTS_KEY" + +# ## Timeout for closing (default: 5s). +# # timeout = "5s" + +# ## Enable additional diagnostic logging. +# enable_diagnostic_logging = false + +# ## Context Tag Sources add Application Insights context tags to a tag value. +# ## +# ## For list of allowed context tag keys see: +# ## https://github.com/Microsoft/ApplicationInsights-Go/blob/master/appinsights/contracts/contexttagkeys.go +# # [outputs.application_insights.context_tag_sources] +# # "ai.cloud.role" = "kubernetes_container_name" +# # "ai.cloud.roleInstance" = "kubernetes_pod_name" +# namepass = ["t.azm.ms/agent_telemetry"] + #tagdrop = ["nodeName"] + ############################################################################### # PROCESSOR PLUGINS # ############################################################################### @@ -119,9 +139,23 @@ [processors.converter.fields] float = ["*"] +############################################################################### +# AGGREGATOR PLUGINS # +############################################################################### +# [[aggregators.quantile]] +# period = "30m" +# drop_original = true +# quantiles = [0.95] +# algorithm = "t-digest" +# compression = 100.0 +# namepass = ["agent_telemetry"] + +############################################################################### +# INPUT PLUGINS # +############################################################################### # Dummy plugin to test out toml parsing happens properly [[inputs.file]] - interval = "24h" + interval = "24h" files = ["test.json"] data_format = "json" @@ -166,3 +200,20 @@ $AZMON_TELEGRAF_CUSTOM_PROM_PLUGINS_WITH_NAMESPACE_FILTER ## OSM Prometheus configuration $AZMON_TELEGRAF_OSM_PROM_PLUGINS + +# [[inputs.procstat]] +# name_prefix="t.azm.ms/" +# exe = "mdsd" +# interval = "60s" +# pid_finder = "native" +# pid_tag = true +# name_override = "agent_telemetry" +# fieldpass = ["cpu_usage", "memory_rss"] +# [inputs.procstat.tags] +# Computer = "$NODE_NAME" +# AgentVersion = "$AGENT_VERSION" +# ControllerType = "$CONTROLLER_TYPE" +# AKS_RESOURCE_ID = "$TELEMETRY_AKS_RESOURCE_ID" +# ACSResourceName = "$TELEMETRY_ACS_RESOURCE_NAME" +# Region = "$TELEMETRY_AKS_REGION" +# ContainerType = "$CONTAINER_TYPE" diff --git a/build/linux/installer/conf/telegraf-rs.conf b/build/linux/installer/conf/telegraf-rs.conf index 5de35d82c..72fc25451 100644 --- a/build/linux/installer/conf/telegraf-rs.conf +++ b/build/linux/installer/conf/telegraf-rs.conf @@ -124,6 +124,26 @@ namedrop = ["agent_telemetry", "file"] #tagdrop = ["AgentVersion","AKS_RESOURCE_ID", "ACS_RESOURCE_NAME", "Region","ClusterName","ClusterType", "Computer", "ControllerType"] +# [[outputs.application_insights]] +# ## Instrumentation key of the Application Insights resource. +# instrumentation_key = "$TELEMETRY_APPLICATIONINSIGHTS_KEY" + +# ## Timeout for closing (default: 5s). +# # timeout = "5s" + +# ## Enable additional diagnostic logging. +# enable_diagnostic_logging = false + +# ## Context Tag Sources add Application Insights context tags to a tag value. +# ## +# ## For list of allowed context tag keys see: +# ## https://github.com/Microsoft/ApplicationInsights-Go/blob/master/appinsights/contracts/contexttagkeys.go +# # [outputs.application_insights.context_tag_sources] +# # "ai.cloud.role" = "kubernetes_container_name" +# # "ai.cloud.roleInstance" = "kubernetes_pod_name" +# namepass = ["t.azm.ms/agent_telemetry"] + #tagdrop = ["nodeName"] + ############################################################################### # PROCESSOR PLUGINS # ############################################################################### @@ -293,6 +313,13 @@ ############################################################################### # AGGREGATOR PLUGINS # ############################################################################### +# [[aggregators.quantile]] +# period = "30m" +# drop_original = true +# quantiles = [0.95] +# algorithm = "t-digest" +# compression = 100.0 +# namepass = ["agent_telemetry"] # # Keep the aggregate basicstats of each metric passing through. # [[aggregators.basicstats]] @@ -369,7 +396,7 @@ # report_active = true # fieldpass = ["usage_active","cluster","node","host","device"] # taginclude = ["cluster","cpu","node"] - + # Read metrics about disk usage by mount point @@ -377,7 +404,7 @@ ## By default stats will be gathered for all mount points. ## Set mount_points will restrict the stats to only the specified mount points. # mount_points = ["/"] - + ## Ignore mount points by filesystem type. # ignore_fs = ["tmpfs", "devtmpfs", "devfs", "overlay", "aufs", "squashfs"] # fieldpass = ["free", "used", "used_percent"] @@ -520,7 +547,7 @@ # Dummy plugin to test out toml parsing happens properly [[inputs.file]] - interval = "24h" + interval = "24h" files = ["test.json"] data_format = "json" @@ -530,10 +557,10 @@ ## An array of urls to scrape metrics from. urls = $AZMON_TELEGRAF_CUSTOM_PROM_URLS - + ## An array of Kubernetes services to scrape metrics from. kubernetes_services = $AZMON_TELEGRAF_CUSTOM_PROM_K8S_SERVICES - + ## Scrape Kubernetes pods for the following prometheus annotations: ## - prometheus.io/scrape: Enable scraping for this pod ## - prometheus.io/scheme: If the metrics endpoint is secured then you will need to @@ -648,3 +675,18 @@ $AZMON_TELEGRAF_OSM_PROM_PLUGINS #[inputs.prometheus.tagpass] # operation_type = ["create_container", "remove_container", "pull_image"] +# [[inputs.procstat]] +# name_prefix="t.azm.ms/" +# exe = "mdsd" +# interval = "60s" +# pid_finder = "native" +# pid_tag = true +# name_override = "agent_telemetry" +# fieldpass = ["cpu_usage", "memory_rss"] +# [inputs.procstat.tags] +# Computer = "$NODE_NAME" +# AgentVersion = "$AGENT_VERSION" +# ControllerType = "$CONTROLLER_TYPE" +# AKS_RESOURCE_ID = "$TELEMETRY_AKS_RESOURCE_ID" +# ACSResourceName = "$TELEMETRY_ACS_RESOURCE_NAME" +# Region = "$TELEMETRY_AKS_REGION" diff --git a/build/linux/installer/conf/telegraf.conf b/build/linux/installer/conf/telegraf.conf index b0a8730c6..9f213e3e8 100644 --- a/build/linux/installer/conf/telegraf.conf +++ b/build/linux/installer/conf/telegraf.conf @@ -158,6 +158,26 @@ namepass = ["container.azm.ms/disk"] #fieldpass = ["used_percent"] +# [[outputs.application_insights]] +# ## Instrumentation key of the Application Insights resource. +# instrumentation_key = "$TELEMETRY_APPLICATIONINSIGHTS_KEY" + +# ## Timeout for closing (default: 5s). +# # timeout = "5s" + +# ## Enable additional diagnostic logging. +# enable_diagnostic_logging = false + + ## Context Tag Sources add Application Insights context tags to a tag value. + ## + ## For list of allowed context tag keys see: + ## https://github.com/Microsoft/ApplicationInsights-Go/blob/master/appinsights/contracts/contexttagkeys.go + # [outputs.application_insights.context_tag_sources] + # "ai.cloud.role" = "kubernetes_container_name" + # "ai.cloud.roleInstance" = "kubernetes_pod_name" + # namepass = ["agent_telemetry"] + #tagdrop = ["nodeName"] + ############################################################################### # PROCESSOR PLUGINS # ############################################################################### @@ -328,7 +348,13 @@ ############################################################################### # AGGREGATOR PLUGINS # ############################################################################### - +# [[aggregators.quantile]] +# period = "30m" +# drop_original = true +# quantiles = [0.95] +# algorithm = "t-digest" +# compression = 100.0 +# namepass = ["t.azm.ms/agent_telemetry"] # # Keep the aggregate basicstats of each metric passing through. # [[aggregators.basicstats]] # ## General Aggregator Arguments: @@ -407,7 +433,7 @@ # Dummy plugin to test out toml parsing happens properly [[inputs.file]] - interval = "24h" + interval = "24h" files = ["test.json"] data_format = "json" @@ -550,14 +576,14 @@ #fieldpass = ["numContainers", "numContainersRunning", "numContainersStopped", "numContainersPaused", "numContainerImages"] # taginclude = ["nodeName"] -#[[inputs.procstat]] -# #name_prefix="t.azm.ms/" +# [[inputs.procstat]] +# name_prefix="t.azm.ms/" # exe = "mdsd" -# interval = "10s" +# interval = "60s" # pid_finder = "native" # pid_tag = true # name_override = "agent_telemetry" -# fieldpass = ["cpu_usage", "memory_rss", "memory_swap", "memory_vms", "memory_stack"] +# fieldpass = ["cpu_usage", "memory_rss"] # [inputs.procstat.tags] # Computer = "$NODE_NAME" # AgentVersion = "$AGENT_VERSION" diff --git a/build/linux/installer/datafiles/base_container.data b/build/linux/installer/datafiles/base_container.data index 985c73a17..9fc7ce08f 100644 --- a/build/linux/installer/datafiles/base_container.data +++ b/build/linux/installer/datafiles/base_container.data @@ -286,6 +286,8 @@ chmod 666 /var/opt/microsoft/docker-cimprov/log/fluent_forward_failed.log touch /var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log chmod 666 /var/opt/microsoft/docker-cimprov/log/arc_k8s_cluster_identity.log +touch /var/opt/microsoft/docker-cimprov/log/appinsights_error.log +chmod 666 /var/opt/microsoft/docker-cimprov/log/appinsights_error.log touch /var/opt/microsoft/docker-cimprov/log/fluentd.log chmod 666 /var/opt/microsoft/docker-cimprov/log/fluentd.log @@ -309,7 +311,7 @@ if ${{PERFORMING_UPGRADE_NOT}}; then rmdir /etc/opt/microsoft/docker-cimprov/conf 2> /dev/null rmdir /etc/opt/microsoft/docker-cimprov 2> /dev/null rmdir /etc/opt/microsoft 2> /dev/null - rmdir /etc/opt 2> /dev/null + rmdir /etc/opt 2> /dev/null fi %Preinstall_0 diff --git a/source/plugins/ruby/ApplicationInsightsUtility.rb b/source/plugins/ruby/ApplicationInsightsUtility.rb index 7691304a6..eb143c4ba 100644 --- a/source/plugins/ruby/ApplicationInsightsUtility.rb +++ b/source/plugins/ruby/ApplicationInsightsUtility.rb @@ -98,6 +98,13 @@ def initializeUtility() elsif !encodedAppInsightsKey.nil? decodedAppInsightsKey = Base64.decode64(encodedAppInsightsKey) + if @@isWindows + logPath = "/etc/omsagentwindows/appinsights_error.log" + else + logPath = "/var/opt/microsoft/docker-cimprov/log/appinsights_error.log" + end + aiLogger = Logger.new(logPath, 1, 2 * 1024 * 1024) + #override ai endpoint if its available otherwise use default. if appInsightsEndpoint && !appInsightsEndpoint.nil? && !appInsightsEndpoint.empty? $log.info("AppInsightsUtility: Telemetry client uses overrided endpoint url : #{appInsightsEndpoint}") @@ -105,20 +112,20 @@ def initializeUtility() #telemetrySynchronousQueue = ApplicationInsights::Channel::SynchronousQueue.new(telemetrySynchronousSender) #telemetryChannel = ApplicationInsights::Channel::TelemetryChannel.new nil, telemetrySynchronousQueue if !isProxyConfigured - sender = ApplicationInsights::Channel::AsynchronousSender.new appInsightsEndpoint + sender = ApplicationInsights::Channel::AsynchronousSender.new appInsightsEndpoint, aiLogger else $log.info("AppInsightsUtility: Telemetry client uses provided proxy configuration since proxy configured") - sender = ApplicationInsights::Channel::AsynchronousSender.new appInsightsEndpoint, @@proxy + sender = ApplicationInsights::Channel::AsynchronousSender.new appInsightsEndpoint, aiLogger, @@proxy end queue = ApplicationInsights::Channel::AsynchronousQueue.new sender channel = ApplicationInsights::Channel::TelemetryChannel.new nil, queue @@Tc = ApplicationInsights::TelemetryClient.new decodedAppInsightsKey, channel else if !isProxyConfigured - sender = ApplicationInsights::Channel::AsynchronousSender.new + sender = ApplicationInsights::Channel::AsynchronousSender.new nil, aiLogger else $log.info("AppInsightsUtility: Telemetry client uses provided proxy configuration since proxy configured") - sender = ApplicationInsights::Channel::AsynchronousSender.new nil, @@proxy + sender = ApplicationInsights::Channel::AsynchronousSender.new nil, aiLogger, @@proxy end queue = ApplicationInsights::Channel::AsynchronousQueue.new sender channel = ApplicationInsights::Channel::TelemetryChannel.new nil, queue diff --git a/source/plugins/ruby/lib/application_insights/channel/asynchronous_sender.rb b/source/plugins/ruby/lib/application_insights/channel/asynchronous_sender.rb index 4786aa1d9..df2138b3a 100644 --- a/source/plugins/ruby/lib/application_insights/channel/asynchronous_sender.rb +++ b/source/plugins/ruby/lib/application_insights/channel/asynchronous_sender.rb @@ -1,5 +1,5 @@ -require_relative 'sender_base' -require 'thread' +require_relative "sender_base" +require "thread" module ApplicationInsights module Channel @@ -17,12 +17,13 @@ module Channel # If no queue items are found for {#send_time} seconds, the worker thread # will shut down (and {#start} will need to be called again). class AsynchronousSender < SenderBase - SERVICE_ENDPOINT_URI = 'https://dc.services.visualstudio.com/v2/track' + SERVICE_ENDPOINT_URI = "https://dc.services.visualstudio.com/v2/track" # Initializes a new instance of the class. # @param [String] service_endpoint_uri the address of the service to send + # @param [Logger] instance of the logger to write the logs (optional) # @param [Hash] proxy server configuration to send (optional) # telemetry data to. - def initialize(service_endpoint_uri = SERVICE_ENDPOINT_URI, proxy = {}) + def initialize(service_endpoint_uri = SERVICE_ENDPOINT_URI, logger = nil, proxy = {}) # callers which requires proxy dont require to maintain service endpoint uri which potentially can change if service_endpoint_uri.nil? || service_endpoint_uri.empty? service_endpoint_uri = SERVICE_ENDPOINT_URI @@ -33,7 +34,7 @@ def initialize(service_endpoint_uri = SERVICE_ENDPOINT_URI, proxy = {}) @lock_work_thread = Mutex.new @work_thread = nil @start_notification_processed = true - super service_endpoint_uri, proxy + super service_endpoint_uri, logger, proxy end # The time span in seconds at which the the worker thread will check the @@ -130,7 +131,7 @@ def run rescue Exception => e # Make sure work_thread sets to nil when it terminates abnormally @work_thread = nil - @logger.error('application_insights') { "Asynchronous sender work thread terminated abnormally: #{e.to_s}" } + @logger.error("application_insights") { "Asynchronous sender work thread terminated abnormally: #{e.to_s}" } end end end diff --git a/source/plugins/ruby/lib/application_insights/channel/sender_base.rb b/source/plugins/ruby/lib/application_insights/channel/sender_base.rb index bedbae4ee..a6b7966db 100644 --- a/source/plugins/ruby/lib/application_insights/channel/sender_base.rb +++ b/source/plugins/ruby/lib/application_insights/channel/sender_base.rb @@ -1,9 +1,9 @@ -require 'yajl/json_gem' -require 'net/http' -require 'openssl' -require 'stringio' -require 'zlib' -require 'logger' +require "yajl/json_gem" +require "net/http" +require "openssl" +require "stringio" +require "zlib" +require "logger" module ApplicationInsights module Channel @@ -16,13 +16,18 @@ module Channel class SenderBase # Initializes a new instance of the class. # @param [String] service_endpoint_uri the address of the service to send + # @param [Logger] instance of the logger to write the logs # @param [Hash] proxy server configuration to send (optional) # telemetry data to. - def initialize(service_endpoint_uri, proxy = {}) + def initialize(service_endpoint_uri, logger, proxy = {}) @service_endpoint_uri = service_endpoint_uri @queue = nil @send_buffer_size = 100 - @logger = Logger.new(STDOUT) + if !logger.nil? && !logger.empty? + @logger = logger + else + @logger = Logger.new(STDOUT) + end @proxy = proxy end @@ -53,9 +58,9 @@ def initialize(service_endpoint_uri, proxy = {}) def send(data_to_send) uri = URI(@service_endpoint_uri) headers = { - 'Accept' => 'application/json', - 'Content-Type' => 'application/json; charset=utf-8', - 'Content-Encoding' => 'gzip' + "Accept" => "application/json", + "Content-Type" => "application/json; charset=utf-8", + "Content-Encoding" => "gzip", } request = Net::HTTP::Post.new(uri.path, headers) @@ -69,7 +74,7 @@ def send(data_to_send) else http = Net::HTTP.new(uri.hostname, uri.port, @proxy[:addr], @proxy[:port], @proxy[:user], @proxy[:pass]) end - if uri.scheme.downcase == 'https' + if uri.scheme.downcase == "https" http.use_ssl = true http.verify_mode = OpenSSL::SSL::VERIFY_PEER end @@ -78,7 +83,7 @@ def send(data_to_send) http.finish if http.started? if !response.kind_of? Net::HTTPSuccess - @logger.warn('application_insights') { "Failed to send data: #{response.message}" } + @logger.warn("application_insights") { "Failed to send data: #{response.message}" } end end diff --git a/source/plugins/ruby/lib/application_insights/channel/synchronous_sender.rb b/source/plugins/ruby/lib/application_insights/channel/synchronous_sender.rb index 597e97b9e..2bb212026 100644 --- a/source/plugins/ruby/lib/application_insights/channel/synchronous_sender.rb +++ b/source/plugins/ruby/lib/application_insights/channel/synchronous_sender.rb @@ -8,14 +8,15 @@ class SynchronousSender < SenderBase SERVICE_ENDPOINT_URI = "https://dc.services.visualstudio.com/v2/track" # Initializes a new instance of the class. # @param [String] service_endpoint_uri the address of the service to send + # @param [Logger] instance of the logger to write the logs (optional) # @param [Hash] proxy server configuration to send (optional) # telemetry data to. - def initialize(service_endpoint_uri = SERVICE_ENDPOINT_URI, proxy = {}) + def initialize(service_endpoint_uri = SERVICE_ENDPOINT_URI, logger = nil, proxy = {}) # callers which requires proxy dont require to maintain service endpoint uri which potentially can change if service_endpoint_uri.nil? || service_endpoint_uri.empty? service_endpoint_uri = SERVICE_ENDPOINT_URI end - super service_endpoint_uri, proxy + super service_endpoint_uri, logger, proxy end end end