microsoft · rashmichandrashekar · Mar 25, 2021 · Dec 3, 2020 · Jan 6, 2021 · Jan 6, 2021
@@ -67,6 +67,7 @@ The general directory structure is:
 │   │   ├── acrworkflows/                     - acr work flows for the Linux Agent container image
 │   │   ├── defaultpromenvvariables           - default environment variables for Prometheus scraping
 │   │   ├── defaultpromenvvariables-rs        - cluster level default environment variables for Prometheus scraping
+│   │   ├── defaultpromenvvariables-sidecar   - cluster level default environment variables for Prometheus scraping in sidecar
 │   ├── windows/                              - scripts to build the Docker image for Windows Agent
 │   │   ├── dockerbuild                       - script to build the code and docker imag, and publish docker image
 │   │   ├── acrworkflows/                     - acr work flows for the Windows Agent container image

@@ -0,0 +1,4 @@
+
+
+
+
@@ -0,0 +1,28 @@
+[SERVICE]
+    #Default service flush interval is 15 seconds
+    Flush         15
+    HTTP_Server   Off
+    Daemon        Off
+    storage.path  /var/opt/microsoft/docker-cimprov/state/flbstore/
+    storage.sync  normal
+    storage.checksum off
+    storage.backlog.mem_limit 10M
+    Log_Level     info
+    Parsers_File  /etc/opt/microsoft/docker-cimprov/azm-containers-parser.conf
+    Log_File      /var/opt/microsoft/docker-cimprov/log/fluent-bit.log
+
+[INPUT]
+    Name        tcp
+    Tag         oms.container.perf.telegraf.*
+    Listen      0.0.0.0
+    Port        25229
+    Chunk_Size  1m
+    Buffer_Size 1m
+    Mem_Buf_Limit 20m
+
+[OUTPUT]
+    Name                            oms
+    EnableTelemetry                 true
+    Retry_Limit                     10
+    TelemetryPushIntervalSeconds    300
+    Match                           oms.container.*
@@ -0,0 +1,162 @@
+# Telegraf Configuration
+#
+# Telegraf is entirely plugin driven. All metrics are gathered from the
+# declared inputs, and sent to the declared outputs.
+#
+# Plugins must be declared in here to be active.
+# To deactivate a plugin, comment out the name and any variables.
+#
+# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
+# file would generate.
+#
+# Environment variables can be used anywhere in this config file, simply prepend
+# them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
+# for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)
+
+
+# Global tags can be specified here in key="value" format.
+[global_tags]
+  hostName = "placeholder_hostname"
+
+# Configuration for telegraf agent
+[agent]
+  ## Default data collection interval for all inputs
+  interval = "60s"
+  ## Rounds collection interval to 'interval'
+  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
+  round_interval = true
+
+  ## Telegraf will send metrics to outputs in batches of at most
+  ## metric_batch_size metrics.
+  ## This controls the size of writes that Telegraf sends to output plugins.
+  metric_batch_size = 3000
+
+  ## For failed writes, telegraf will cache metric_buffer_limit metrics for each
+  ## output, and will flush this buffer on a successful write. Oldest metrics
+  ## are dropped first when this buffer fills.
+  ## This buffer only fills when writes fail to output plugin(s).
+  metric_buffer_limit = 60000
+
+  ## Collection jitter is used to jitter the collection by a random amount.
+  ## Each plugin will sleep for a random time within jitter before collecting.
+  ## This can be used to avoid many plugins querying things like sysfs at the
+  ## same time, which can have a measurable effect on the system.
+  collection_jitter = "0s"
+
+  ## Default flushing interval for all outputs. You shouldn't set this below
+  ## interval. Maximum flush_interval will be flush_interval + flush_jitter
+  flush_interval = "15s"
+  ## Jitter the flush interval by a random amount. This is primarily to avoid
+  ## large write spikes for users running a large number of telegraf instances.
+  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
+  flush_jitter = "0s"
+
+  ## By default or when set to "0s", precision will be set to the same
+  ## timestamp order as the collection interval, with the maximum being 1s.
+  ##   ie, when interval = "10s", precision will be "1s"
+  ##       when interval = "250ms", precision will be "1ms"
+  ## Precision will NOT be used for service inputs. It is up to each individual
+  ## service input to set the timestamp at the appropriate precision.
+  ## Valid time units are "ns", "us" (or "µs"), "ms", "s".
+  precision = ""
+
+  ## Logging configuration:
+  ## Run telegraf with debug log messages.
+  debug = false
+  ## Run telegraf in quiet mode (error log messages only).
+  quiet = true
+  ## Specify the log file name. The empty string means to log to stderr.
+  logfile = ""
+  ## Override default hostname, if empty use os.Hostname()
+  #hostname = "placeholder_hostname"
+  ## If set to true, do no set the "host" tag in the telegraf agent.
+  omit_hostname = true
+
+
+###############################################################################
+#                            OUTPUT PLUGINS                                   #
+###############################################################################
+
+# Generic socket writer capable of handling multiple socket types.
+[[outputs.socket_writer]]
+  ## URL to connect to
+  address = "tcp://0.0.0.0:25229"
+  # address = "tcp://example.com:http"
+  # address = "tcp4://127.0.0.1:8094"
+  # address = "tcp6://127.0.0.1:8094"
+  # address = "tcp6://[2001:db8::1]:8094"
+  # address = "udp://127.0.0.1:8094"
+  # address = "udp4://127.0.0.1:8094"
+  # address = "udp6://127.0.0.1:8094"
+  # address = "unix:///tmp/telegraf.sock"
+  # address = "unixgram:///tmp/telegraf.sock"
+
+  ## Optional TLS Config
+  # tls_ca = "/etc/telegraf/ca.pem"
+  # tls_cert = "/etc/telegraf/cert.pem"
+  # tls_key = "/etc/telegraf/key.pem"
+  ## Use TLS but skip chain & host verification
+  # insecure_skip_verify = false
+
+  ## Period between keep alive probes.
+  ## Only applies to TCP sockets.
+  ## 0 disables keep alive probes.
+  ## Defaults to the OS configuration.
+  # keep_alive_period = "5m"
+
+  ## Data format to generate.
+  ## Each data format has its own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+  data_format = "json"
+  namedrop = ["agent_telemetry"]
+
+###############################################################################
+#                            PROCESSOR PLUGINS                                #
+###############################################################################
+
+[[processors.converter]]
+  [processors.converter.fields]
+    float = ["*"]
+
+#Prometheus Custom Metrics
+[[inputs.prometheus]]
+  interval = "$AZMON_TELEGRAF_CUSTOM_PROM_INTERVAL"
+
+  ## Scrape Kubernetes pods for the following prometheus annotations:
+  ## - prometheus.io/scrape: Enable scraping for this pod
+  ## - prometheus.io/scheme: If the metrics endpoint is secured then you will need to
+  ##     set this to `https` & most likely set the tls config.
+  ## - prometheus.io/path: If the metrics path is not /metrics, define it with this annotation.
+  ## - prometheus.io/port: If port is not 9102 use this annotation
+  $AZMON_TELEGRAF_CUSTOM_PROM_MONITOR_PODS
+  $AZMON_TELEGRAF_CUSTOM_PROM_SCRAPE_SCOPE
+
+  $AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_LABEL_SELECTOR
+  $AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_FIELD_SELECTOR
+
+  fieldpass = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDPASS
+  fielddrop = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDDROP
+
+  metric_version = 2
+  url_tag = "scrapeUrl"
+  ## Kubernetes config file to create client from.
+  # kube_config = "/path/to/kubernetes.config"
+
+  ## Use bearer token for authorization. ('bearer_token' takes priority)
+  bearer_token = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+  ## OR
+  # bearer_token_string = "abc_123"
+
+  ## Specify timeout duration for slower prometheus clients (default is 3s)
+  response_timeout = "15s"
+
+  ## Optional TLS Config
+  tls_ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  ## Use TLS but skip chain & host verification
+  insecure_skip_verify = true
+
+$AZMON_TELEGRAF_CUSTOM_PROM_PLUGINS_WITH_NAMESPACE_FILTER
+
+## OSM Prometheus configuration
+$AZMON_TELEGRAF_OSM_PROM_PLUGINS
@@ -540,24 +540,29 @@
 
 #Prometheus Custom Metrics
 [[inputs.prometheus]]
-  interval = "$AZMON_RS_PROM_INTERVAL"
+  interval = "$AZMON_TELEGRAF_CUSTOM_PROM_INTERVAL"
 
   ## An array of urls to scrape metrics from.
-  urls = $AZMON_RS_PROM_URLS
+  urls = $AZMON_TELEGRAF_CUSTOM_PROM_URLS
 
   ## An array of Kubernetes services to scrape metrics from.
-  kubernetes_services = $AZMON_RS_PROM_K8S_SERVICES
+  kubernetes_services = $AZMON_TELEGRAF_CUSTOM_PROM_K8S_SERVICES
 
   ## Scrape Kubernetes pods for the following prometheus annotations:
   ## - prometheus.io/scrape: Enable scraping for this pod
   ## - prometheus.io/scheme: If the metrics endpoint is secured then you will need to
   ##     set this to `https` & most likely set the tls config.
   ## - prometheus.io/path: If the metrics path is not /metrics, define it with this annotation.
   ## - prometheus.io/port: If port is not 9102 use this annotation
-  $AZMON_RS_PROM_MONITOR_PODS
+  $AZMON_TELEGRAF_CUSTOM_PROM_MONITOR_PODS
 
-  fieldpass = $AZMON_RS_PROM_FIELDPASS
-  fielddrop = $AZMON_RS_PROM_FIELDDROP
+  $AZMON_TELEGRAF_CUSTOM_PROM_SCRAPE_SCOPE
+
+  $AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_LABEL_SELECTOR
+  $AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_FIELD_SELECTOR
+
+  fieldpass = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDPASS
+  fielddrop = $AZMON_TELEGRAF_CUSTOM_PROM_FIELDDROP
 
   metric_version = 2
   url_tag = "scrapeUrl"
@@ -581,7 +586,11 @@
   insecure_skip_verify = true
   #tagexclude = ["AgentVersion","AKS_RESOURCE_ID","ACS_RESOURCE_NAME", "Region", "ClusterName", "ClusterType", "Computer", "ControllerType"]
 
-$AZMON_RS_PROM_PLUGINS_WITH_NAMESPACE_FILTER
+$AZMON_TELEGRAF_CUSTOM_PROM_PLUGINS_WITH_NAMESPACE_FILTER
+
+## OSM Prometheus configuration
+$AZMON_TELEGRAF_OSM_PROM_PLUGINS
+
 # [[inputs.exec]]
 #   ## Commands array
 #   interval = "15m"

@@ -110,24 +110,28 @@ MAINTAINER:              'Microsoft Corporation'
 /opt/tomlrb/string_utils.rb;                                    source/toml-parser/tomlrb/string_utils.rb; 644; root; root
 /opt/tomlrb/version.rb;                                         source/toml-parser/tomlrb/version.rb; 644; root; root
 
-/opt/td-agent-bit/bin/out_oms.so;                               intermediate/${{BUILD_CONFIGURATION}}/out_oms.so;  	755; root; root
-/etc/opt/microsoft/docker-cimprov/td-agent-bit.conf;			    build/linux/installer/conf/td-agent-bit.conf;                    644; root; root
-/etc/opt/microsoft/docker-cimprov/td-agent-bit-rs.conf;			 build/linux/installer/conf/td-agent-bit-rs.conf;                    644; root; root
-/etc/opt/microsoft/docker-cimprov/azm-containers-parser.conf;	 build/linux/installer/conf/azm-containers-parser.conf;                    644; root; root
-/etc/opt/microsoft/docker-cimprov/out_oms.conf;			          build/linux/installer/conf/out_oms.conf;                    644; root; root
-/etc/opt/microsoft/docker-cimprov/telegraf.conf;			       build/linux/installer/conf/telegraf.conf;                    644; root; root
-/etc/opt/microsoft/docker-cimprov/telegraf-rs.conf;			    build/linux/installer/conf/telegraf-rs.conf;                    644; root; root
-/opt/microsoft/docker-cimprov/bin/TelegrafTCPErrorTelemetry.sh; build/linux/installer/scripts/TelegrafTCPErrorTelemetry.sh;      755; root; root
-/opt/livenessprobe.sh;                                          build/linux/installer/scripts/livenessprobe.sh;      755; root; root
-/opt/tomlparser-prom-customconfig.rb;                           build/linux/installer/scripts/tomlparser-prom-customconfig.rb;     755; root; root
-/opt/tomlparser-mdm-metrics-config.rb;                          build/linux/installer/scripts/tomlparser-mdm-metrics-config.rb;     755; root; root
-/opt/tomlparser-metric-collection-config.rb;                    build/linux/installer/scripts/tomlparser-metric-collection-config.rb;     755; root; root
+/opt/td-agent-bit/bin/out_oms.so;                                       intermediate/${{BUILD_CONFIGURATION}}/out_oms.so;  	755; root; root
+/etc/opt/microsoft/docker-cimprov/prometheus-side-car.conf;		         build/linux/installer/conf/prometheus-side-car.conf;                    644; root; root
+/etc/opt/microsoft/docker-cimprov/td-agent-bit.conf;			            build/linux/installer/conf/td-agent-bit.conf;                    644; root; root
+/etc/opt/microsoft/docker-cimprov/td-agent-bit-prom-side-car.conf;	   build/linux/installer/conf/td-agent-bit-prom-side-car.conf;                    644; root; root
+/etc/opt/microsoft/docker-cimprov/td-agent-bit-rs.conf;			         build/linux/installer/conf/td-agent-bit-rs.conf;                    644; root; root
+/etc/opt/microsoft/docker-cimprov/azm-containers-parser.conf;	         build/linux/installer/conf/azm-containers-parser.conf;                    644; root; root
+/etc/opt/microsoft/docker-cimprov/out_oms.conf;			                  build/linux/installer/conf/out_oms.conf;                    644; root; root
+/etc/opt/microsoft/docker-cimprov/telegraf.conf;			               build/linux/installer/conf/telegraf.conf;                    644; root; root
+/etc/opt/microsoft/docker-cimprov/telegraf-prom-side-car.conf;			   build/linux/installer/conf/telegraf-prom-side-car.conf;                    644; root; root
+/etc/opt/microsoft/docker-cimprov/telegraf-rs.conf;			            build/linux/installer/conf/telegraf-rs.conf;                    644; root; root
+/opt/microsoft/docker-cimprov/bin/TelegrafTCPErrorTelemetry.sh;         build/linux/installer/scripts/TelegrafTCPErrorTelemetry.sh;      755; root; root
+/opt/livenessprobe.sh;                                                  build/linux/installer/scripts/livenessprobe.sh;      755; root; root
+/opt/tomlparser-prom-customconfig.rb;                                   build/common/installer/scripts/tomlparser-prom-customconfig.rb;     755; root; root
+/opt/tomlparser-mdm-metrics-config.rb;                                  build/linux/installer/scripts/tomlparser-mdm-metrics-config.rb;     755; root; root
+/opt/tomlparser-metric-collection-config.rb;                            build/linux/installer/scripts/tomlparser-metric-collection-config.rb;     755; root; root
 
 /opt/tomlparser-agent-config.rb;                                build/linux/installer/scripts/tomlparser-agent-config.rb;     755; root; root
 /opt/tomlparser.rb;                                             build/common/installer/scripts/tomlparser.rb;     755; root; root
 /opt/td-agent-bit-conf-customizer.rb;                           build/common/installer/scripts/td-agent-bit-conf-customizer.rb;     755; root; root
 /opt/ConfigParseErrorLogger.rb;                                 build/common/installer/scripts/ConfigParseErrorLogger.rb;           755; root; root
 /opt/tomlparser-npm-config.rb;                                  build/linux/installer/scripts/tomlparser-npm-config.rb;     755; root; root
+/opt/tomlparser-osm-config.rb;                                  build/linux/installer/scripts/tomlparser-osm-config.rb;     755; root; root
 
 
 /opt/microsoft/omsagent/plugin/filter_cadvisor_health_container.rb;                                 source/plugins/ruby/filter_cadvisor_health_container.rb; 644; root; root

@@ -26,15 +26,22 @@ then
  exit 1
 fi
 
-if [ ! -s "inotifyoutput.txt" ]
+if [ -s "inotifyoutput.txt" ]
 then
-  # inotifyoutput file is empty and the grep commands for omsagent and td-agent-bit succeeded
-  exit 0
-else
-  if [ -s "inotifyoutput.txt" ]
-  then
-    # inotifyoutput file has data(config map was applied)
-    echo "inotifyoutput.txt has been updated - config changed" > /dev/termination-log
-    exit 1
-  fi
+  # inotifyoutput file has data(config map was applied)
+  echo "inotifyoutput.txt has been updated - config changed" > /dev/termination-log
+  exit 1
 fi
+
+# Perform the following check only for prometheus sidecar that does OSM scraping or for replicaset when sidecar scraping is disabled
+if [[ ( ( ! -e "/etc/config/kube.conf" ) && ( "${CONTAINER_TYPE}" == "PrometheusSidecar" ) ) ||
+      ( ( -e "/etc/config/kube.conf" ) && ( ( ! -z "${SIDECAR_SCRAPING_ENABLED}" ) && ( "${SIDECAR_SCRAPING_ENABLED}" == "false" ) ) ) ]]; then
+    if [ -s "inotifyoutput-osm.txt" ]
+    then
+      # inotifyoutput-osm file has data(config map was applied)
+      echo "inotifyoutput-osm.txt has been updated - config changed" > /dev/termination-log
+      exit 1
+    fi
+fi
+
+exit 0