microsoft · rashmichandrashekar · Jul 9, 2019 · Jun 19, 2019 · Jun 21, 2019 · Jun 21, 2019
diff --git a/installer/conf/td-agent-bit-rs.conf b/installer/conf/td-agent-bit-rs.conf
@@ -4,16 +4,6 @@
     Parsers_File  /etc/td-agent-bit/parsers.conf
     Log_File      /var/opt/microsoft/docker-cimprov/log/fluent-bit.log
 
-[INPUT]
-    Name tail
-    Tag oms.container.log.telegraf.err.*
-    Path /var/opt/microsoft/docker-cimprov/log/telegraf.log
-    DB /var/opt/microsoft/docker-cimprov/state/telegraf-log-state.db
-    Mem_Buf_Limit 2m
-    Path_Key filepath
-    Skip_Long_Lines On
-    Ignore_Older 5m
-
 [INPUT]
     Name        tcp
     Tag oms.container.perf.telegraf.*

diff --git a/installer/conf/td-agent-bit.conf b/installer/conf/td-agent-bit.conf
@@ -6,7 +6,7 @@
 
 [INPUT]
     Name tail
-    Tag oms.container.log.*
+    Tag oms.container.log.la.*
     Path ${AZMON_LOG_TAIL_PATH}
     DB /var/log/omsagent-fblogs.db
     DB.Sync Off
@@ -32,17 +32,6 @@
     Skip_Long_Lines On
     Ignore_Older 2m
 
-[INPUT]
-    Name tail
-    Tag oms.container.log.telegraf.err.*
-    Path /var/opt/microsoft/docker-cimprov/log/telegraf.log
-    DB /var/opt/microsoft/docker-cimprov/state/telegraf-log-state.db
-    DB.Sync Off
-    Mem_Buf_Limit 1m
-    Path_Key filepath
-    Skip_Long_Lines On
-    Ignore_Older 2m
-
 [INPUT]
     Name        tcp
     Tag oms.container.perf.telegraf.*
@@ -53,9 +42,16 @@
 
 [FILTER]
     Name grep
-    Match oms.container.log.*
+    Match oms.container.log.la.*
     Exclude stream ${AZMON_LOG_EXCLUSION_REGEX_PATTERN}
 
+# Exclude prometheus plugin exceptions that might be caused due to invalid config.(Logs which contain - E! [inputs.prometheus])
+# Excluding these logs from being sent to AI since it can result in high volume of data in telemetry due to invalid config.
+[FILTER]
+    Name grep
+    Match oms.container.log.flbplugin.*
+    Exclude log E! [\[]inputs.prometheus[\]]
+
 [OUTPUT]
     Name                            oms
     EnableTelemetry                 true

diff --git a/installer/conf/telegraf-rs.conf b/installer/conf/telegraf-rs.conf
@@ -77,7 +77,7 @@
   ## Run telegraf in quiet mode (error log messages only).
   quiet = true
   ## Specify the log file name. The empty string means to log to stderr.
-  logfile = "/var/opt/microsoft/docker-cimprov/log/telegraf.log"
+  logfile = ""
 
   ## Override default hostname, if empty use os.Hostname()
   #hostname = "placeholder_hostname"
@@ -536,32 +536,75 @@
   #tagexclude = ["AgentVersion","AKS_RESOURCE_ID","ACS_RESOURCE_NAME", "Region", "ClusterName", "ClusterType", "Computer", "ControllerType"]
 #  [inputs.prometheus.tagpass]
 
-[[inputs.exec]]
-  ## Commands array
-  interval = "15m"
-  commands = [
-    "/opt/microsoft/docker-cimprov/bin/TelegrafTCPErrorTelemetry.sh"
-  ]
+#Prometheus Custom Metrics
+[[inputs.prometheus]]
+  interval = "$AZMON_RS_PROM_INTERVAL"
 
-  ## Timeout for each command to complete.
-  timeout = "15s"
+  ## An array of urls to scrape metrics from.
+  urls = $AZMON_RS_PROM_URLS
+
+  ## An array of Kubernetes services to scrape metrics from.
+  kubernetes_services = $AZMON_RS_PROM_K8S_SERVICES
+
+  ## Scrape Kubernetes pods for the following prometheus annotations:
+  ## - prometheus.io/scrape: Enable scraping for this pod
+  ## - prometheus.io/scheme: If the metrics endpoint is secured then you will need to
+  ##     set this to `https` & most likely set the tls config.
+  ## - prometheus.io/path: If the metrics path is not /metrics, define it with this annotation.
+  ## - prometheus.io/port: If port is not 9102 use this annotation
+  monitor_kubernetes_pods = $AZMON_RS_PROM_MONITOR_PODS
 
-  ## measurement name suffix (for separating different commands)
-  name_suffix = "_telemetry"
+  fieldpass = $AZMON_RS_PROM_FIELDPASS
+  fielddrop = $AZMON_RS_PROM_FIELDDROP
 
-  ## Data format to consume.
-  ## Each data format has its own unique set of configuration options, read
-  ## more about them here:
-  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
-  data_format = "influx"
-  #tagexclude = ["hostName"]
-  [inputs.exec.tags]
-    AgentVersion = "$AGENT_VERSION"
-    AKS_RESOURCE_ID = "$TELEMETRY_AKS_RESOURCE_ID"
-    ACS_RESOURCE_NAME = "$TELEMETRY_ACS_RESOURCE_NAME"
-    Region = "$TELEMETRY_AKS_REGION"
-    ClusterName = "$TELEMETRY_CLUSTER_NAME"
-    ClusterType = "$TELEMETRY_CLUSTER_TYPE"
-    Computer = "placeholder_hostname"
-    ControllerType = "$CONTROLLER_TYPE"
+  metric_version = 2
+  url_tag = "scrapeUrl"
+
+  ## Kubernetes config file to create client from.
+  # kube_config = "/path/to/kubernetes.config"
+
+  ## Use bearer token for authorization. ('bearer_token' takes priority)
+  bearer_token = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+  ## OR
+  # bearer_token_string = "abc_123"
+
+  ## Specify timeout duration for slower prometheus clients (default is 3s)
+  response_timeout = "15s"
+
+  ## Optional TLS Config
+  tls_ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  #tls_cert = /path/to/certfile
+  # tls_key = /path/to/keyfile
+  ## Use TLS but skip chain & host verification
+  insecure_skip_verify = true
+  #tagexclude = ["AgentVersion","AKS_RESOURCE_ID","ACS_RESOURCE_NAME", "Region", "ClusterName", "ClusterType", "Computer", "ControllerType"]
+
+# [[inputs.exec]]
+#   ## Commands array
+#   interval = "15m"
+#   commands = [
+#     "/opt/microsoft/docker-cimprov/bin/TelegrafTCPErrorTelemetry.sh"
+#   ]
+
+#   ## Timeout for each command to complete.
+#   timeout = "15s"
+
+#   ## measurement name suffix (for separating different commands)
+#   name_suffix = "_telemetry"
+
+#   ## Data format to consume.
+#   ## Each data format has its own unique set of configuration options, read
+#   ## more about them here:
+#   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+#   data_format = "influx"
+#   #tagexclude = ["hostName"]
+#   [inputs.exec.tags]
+#     AgentVersion = "$AGENT_VERSION"
+#     AKS_RESOURCE_ID = "$TELEMETRY_AKS_RESOURCE_ID"
+#     ACS_RESOURCE_NAME = "$TELEMETRY_ACS_RESOURCE_NAME"
+#     Region = "$TELEMETRY_AKS_REGION"
+#     ClusterName = "$TELEMETRY_CLUSTER_NAME"
+#     ClusterType = "$TELEMETRY_CLUSTER_TYPE"
+#     Computer = "placeholder_hostname"
+#     ControllerType = "$CONTROLLER_TYPE"
 
diff --git a/installer/conf/telegraf.conf b/installer/conf/telegraf.conf
@@ -77,8 +77,7 @@
   ## Run telegraf in quiet mode (error log messages only).
   quiet = true
   ## Specify the log file name. The empty string means to log to stderr.
-  logfile = "/var/opt/microsoft/docker-cimprov/log/telegraf.log"
-
+  logfile = ""
   ## Override default hostname, if empty use os.Hostname()
   #hostname = "placeholder_hostname"
   ## If set to true, do no set the "host" tag in the telegraf agent.
@@ -568,31 +567,66 @@
   insecure_skip_verify = true
   #tagexclude = ["AgentVersion","AKS_RESOURCE_ID","ACS_RESOURCE_NAME", "Region", "ClusterName", "ClusterType", "Computer", "ControllerType"]
 
-[[inputs.exec]]
-  ## Commands array
-  interval = "15m"
-  commands = [
-    "/opt/microsoft/docker-cimprov/bin/TelegrafTCPErrorTelemetry.sh"
-  ]
 
-  ## Timeout for each command to complete.
-  timeout = "15s"
+## prometheus custom metrics
+[[inputs.prometheus]]
 
-  ## measurement name suffix (for separating different commands)
-  name_suffix = "_telemetry"
+  interval = "$AZMON_DS_PROM_INTERVAL"
 
-  ## Data format to consume.
-  ## Each data format has its own unique set of configuration options, read
-  ## more about them here:
-  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
-  data_format = "influx"
-  tagexclude = ["hostName"]
-  [inputs.exec.tags]
-    AgentVersion = "$AGENT_VERSION"
-    AKS_RESOURCE_ID = "$TELEMETRY_AKS_RESOURCE_ID"
-    ACS_RESOURCE_NAME = "$TELEMETRY_ACS_RESOURCE_NAME"
-    Region = "$TELEMETRY_AKS_REGION"
-    ClusterName = "$TELEMETRY_CLUSTER_NAME"
-    ClusterType = "$TELEMETRY_CLUSTER_TYPE"
-    Computer = "placeholder_hostname"
-    ControllerType = "$CONTROLLER_TYPE"
+  ## An array of urls to scrape metrics from.
+  urls = $AZMON_DS_PROM_URLS
+
+  fieldpass = $AZMON_DS_PROM_FIELDPASS
+
+  fielddrop = $AZMON_DS_PROM_FIELDDROP
+
+  metric_version = 2
+  url_tag = "scrapeUrl"
+
+  ## Kubernetes config file to create client from.
+  # kube_config = "/path/to/kubernetes.config"
+
+  ## Use bearer token for authorization. ('bearer_token' takes priority)
+  bearer_token = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+  ## OR
+  # bearer_token_string = "abc_123"
+
+  ## Specify timeout duration for slower prometheus clients (default is 3s)
+  response_timeout = "15s"
+
+  ## Optional TLS Config
+  tls_ca = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+  #tls_cert = /path/to/certfile
+  # tls_key = /path/to/keyfile
+  ## Use TLS but skip chain & host verification
+  insecure_skip_verify = true
+  #tagexclude = ["AgentVersion","AKS_RESOURCE_ID","ACS_RESOURCE_NAME", "Region", "ClusterName", "ClusterType", "Computer", "ControllerType"]
+
+# [[inputs.exec]]
+#   ## Commands array
+#   interval = "15m"
+#   commands = [
+#     "/opt/microsoft/docker-cimprov/bin/TelegrafTCPErrorTelemetry.sh"
+#   ]
+
+#   ## Timeout for each command to complete.
+#   timeout = "15s"
+
+#   ## measurement name suffix (for separating different commands)
+#   name_suffix = "_telemetry"
+
+#   ## Data format to consume.
+#   ## Each data format has its own unique set of configuration options, read
+#   ## more about them here:
+#   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+#   data_format = "influx"
+#   tagexclude = ["hostName"]
+#   [inputs.exec.tags]
+#     AgentVersion = "$AGENT_VERSION"
+#     AKS_RESOURCE_ID = "$TELEMETRY_AKS_RESOURCE_ID"
+#     ACS_RESOURCE_NAME = "$TELEMETRY_ACS_RESOURCE_NAME"
+#     Region = "$TELEMETRY_AKS_REGION"
+#     ClusterName = "$TELEMETRY_CLUSTER_NAME"
+#     ClusterType = "$TELEMETRY_CLUSTER_TYPE"
+#     Computer = "placeholder_hostname"
+#     ControllerType = "$CONTROLLER_TYPE"
diff --git a/installer/datafiles/base_container.data b/installer/datafiles/base_container.data
@@ -110,9 +110,10 @@ MAINTAINER:              'Microsoft Corporation'
 /etc/opt/microsoft/docker-cimprov/out_oms.conf;			            installer/conf/out_oms.conf;                    644; root; root
 /etc/opt/microsoft/docker-cimprov/telegraf.conf;			        installer/conf/telegraf.conf;                    644; root; root
 /etc/opt/microsoft/docker-cimprov/telegraf-rs.conf;			        installer/conf/telegraf-rs.conf;                    644; root; root
-/opt/microsoft/docker-cimprov/bin/TelegrafTCPErrorTelemetry.sh;	        installer/scripts/TelegrafTCPErrorTelemetry.sh;      755; root; root
+/opt/microsoft/docker-cimprov/bin/TelegrafTCPErrorTelemetry.sh;	    installer/scripts/TelegrafTCPErrorTelemetry.sh;      755; root; root
 /opt/livenessprobe.sh;                                              installer/scripts/livenessprobe.sh;      755; root; root
 /opt/tomlparser.rb;                                                 installer/scripts/tomlparser.rb;     755; root; root 
+/opt/tomlparser-prom-customconfig.rb;                               installer/scripts/tomlparser-prom-customconfig.rb;     755; root; root 
 
 %Links
 /opt/omi/lib/libcontainer.${{SHLIB_EXT}}; /opt/microsoft/docker-cimprov/lib/libcontainer.${{SHLIB_EXT}}; 644; root; root