Changes for creating custom plugins with namespace settings for prometheus scraping

installer/conf/telegraf-rs.conf

@@ -552,7 +552,7 @@
   ##     set this to `https` & most likely set the tls config.
   ## - prometheus.io/path: If the metrics path is not /metrics, define it with this annotation.
   ## - prometheus.io/port: If port is not 9102 use this annotation
-  monitor_kubernetes_pods = $AZMON_RS_PROM_MONITOR_PODS
+  $AZMON_RS_PROM_MONITOR_PODS
 
   fieldpass = $AZMON_RS_PROM_FIELDPASS
   fielddrop = $AZMON_RS_PROM_FIELDDROP
@@ -579,6 +579,7 @@
   insecure_skip_verify = true
   #tagexclude = ["AgentVersion","AKS_RESOURCE_ID","ACS_RESOURCE_NAME", "Region", "ClusterName", "ClusterType", "Computer", "ControllerType"]
 
+$AZMON_RS_PROM_PLUGINS_WITH_NAMESPACE_FILTER
 # [[inputs.exec]]
 #   ## Commands array
 #   interval = "15m"

installer/scripts/tomlparser-prom-customconfig.rb

@@ -18,6 +18,14 @@
 @defaultRsK8sServices = []
 @defaultRsMonitorPods = false
 
+#Configurations to be used for the auto-generated input prometheus plugins for namespace filtering
+@metricVersion = 2
+@urlTag = "scrapeUrl"
+@bearerToken = "/var/run/secrets/kubernetes.io/serviceaccount/token"
+@responseTimeout = "15s"
+@tlsCa = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+@insecureSkipVerify = true
+
 # Use parser to parse the configmap toml file to a ruby structure
 def parseConfigMap
   begin
@@ -53,6 +61,48 @@ def checkForType(variable, varType)
   end
 end
 
+def replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods)
+  begin
+    new_contents = new_contents.gsub("$AZMON_RS_PROM_MONITOR_PODS", ("monitor_kubernetes_pods = #{monitorKubernetesPods}"))
+    new_contents = new_contents.gsub("$AZMON_RS_PROM_PLUGINS_WITH_NAMESPACE_FILTER", "")
+  rescue => errorStr
+    puts "config::error::Exception while replacing default pod monitor settings: #{errorStr}"
+  end
+  return new_contents
+end
+
+def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKubernetesPodsNamespaces, new_contents, interval, fieldPassSetting, fieldDropSetting)
+  begin
+    new_contents = new_contents.gsub("$AZMON_RS_PROM_MONITOR_PODS", "# Commenting this out since new plugins will be created per namespace\n  # $AZMON_RS_PROM_MONITOR_PODS")
+    pluginConfigsWithNamespaces = ""
+    monitorKubernetesPodsNamespaces.each do |namespace|
+      if !namespace.nil?
+        #Stripping namespaces to remove leading and trailing whitespaces
+        namespace.strip!
+        if namespace.length > 0
+          pluginConfigsWithNamespaces += "\n[[inputs.prometheus]]
+  interval = \"#{interval}\"
+  monitor_kubernetes_pods = true
+  monitor_kubernetes_pods_namespace = \"#{namespace}\"
+  fieldpass = #{fieldPassSetting}
+  fielddrop = #{fieldDropSetting}
+  metric_version = #{@metricVersion}
+  url_tag = \"#{@urlTag}\"
+  bearer_token = \"#{@bearerToken}\"
+  response_timeout = \"#{@responseTimeout}\"
+  tls_ca = \"#{@tlsCa}\"
+  insecure_skip_verify = #{@insecureSkipVerify}\n"
+        end
+      end
+    end
+    new_contents = new_contents.gsub("$AZMON_RS_PROM_PLUGINS_WITH_NAMESPACE_FILTER", pluginConfigsWithNamespaces)
+    return new_contents
+  rescue => errorStr
+    puts "config::error::Exception while creating prometheus input plugins to filter namespaces: #{errorStr}, using defaults"
+    replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods)
+  end
+end
+
 # Use the ruby structure created after config parsing to set the right values to be used as environment variables
 def populateSettingValuesFromConfigMap(parsedConfig)
   # Checking to see if this is the daemonset or replicaset to parse config accordingly
@@ -68,22 +118,23 @@ def populateSettingValuesFromConfigMap(parsedConfig)
           urls = parsedConfig[:prometheus_data_collection_settings][:cluster][:urls]
           kubernetesServices = parsedConfig[:prometheus_data_collection_settings][:cluster][:kubernetes_services]
           monitorKubernetesPods = parsedConfig[:prometheus_data_collection_settings][:cluster][:monitor_kubernetes_pods]
+          monitorKubernetesPodsNamespaces = parsedConfig[:prometheus_data_collection_settings][:cluster][:monitor_kubernetes_pods_namespaces]
 
           # Check for the right datattypes to enforce right setting values
           if checkForType(interval, String) &&
              checkForTypeArray(fieldPass, String) &&
              checkForTypeArray(fieldDrop, String) &&
              checkForTypeArray(kubernetesServices, String) &&
              checkForTypeArray(urls, String) &&
-             !monitorKubernetesPods.nil? && (!!monitorKubernetesPods == monitorKubernetesPods) #Checking for Boolean type, since 'Boolean' is not defined as a type in ruby
+             (monitorKubernetesPods.nil? || (!monitorKubernetesPods.nil? && (!!monitorKubernetesPods == monitorKubernetesPods))) #Checking for Boolean type, since 'Boolean' is not defined as a type in ruby
             puts "config::Successfully passed typecheck for config settings for replicaset"
             #if setting is nil assign default values
             interval = (interval.nil?) ? @defaultRsInterval : interval
             fieldPass = (fieldPass.nil?) ? @defaultRsFieldPass : fieldPass
             fieldDrop = (fieldDrop.nil?) ? @defaultRsFieldDrop : fieldDrop
             kubernetesServices = (kubernetesServices.nil?) ? @defaultRsK8sServices : kubernetesServices
             urls = (urls.nil?) ? @defaultRsPromUrls : urls
-            monitorKubernetesPods = (kubernetesServices.nil?) ? @defaultRsMonitorPods : monitorKubernetesPods
+            monitorKubernetesPods = (monitorKubernetesPods.nil?) ? @defaultRsMonitorPods : monitorKubernetesPods
 
             file_name = "/opt/telegraf-test-rs.conf"
             # Copy the telegraf config file to a temp file to run telegraf in test mode with this config
@@ -93,11 +144,24 @@ def populateSettingValuesFromConfigMap(parsedConfig)
             #Replace the placeholder config values with values from custom config
             text = File.read(file_name)
             new_contents = text.gsub("$AZMON_RS_PROM_INTERVAL", interval)
-            new_contents = new_contents.gsub("$AZMON_RS_PROM_FIELDPASS", ((fieldPass.length > 0) ? ("[\"" + fieldPass.join("\",\"") + "\"]") : "[]"))
-            new_contents = new_contents.gsub("$AZMON_RS_PROM_FIELDDROP", ((fieldDrop.length > 0) ? ("[\"" + fieldDrop.join("\",\"") + "\"]") : "[]"))
+            fieldPassSetting = (fieldPass.length > 0) ? ("[\"" + fieldPass.join("\",\"") + "\"]") : "[]"
+            new_contents = new_contents.gsub("$AZMON_RS_PROM_FIELDPASS", fieldPassSetting)
+            fieldDropSetting = (fieldDrop.length > 0) ? ("[\"" + fieldDrop.join("\",\"") + "\"]") : "[]"
+            new_contents = new_contents.gsub("$AZMON_RS_PROM_FIELDDROP", fieldDropSetting)
             new_contents = new_contents.gsub("$AZMON_RS_PROM_URLS", ((urls.length > 0) ? ("[\"" + urls.join("\",\"") + "\"]") : "[]"))
             new_contents = new_contents.gsub("$AZMON_RS_PROM_K8S_SERVICES", ((kubernetesServices.length > 0) ? ("[\"" + kubernetesServices.join("\",\"") + "\"]") : "[]"))
-            new_contents = new_contents.gsub("$AZMON_RS_PROM_MONITOR_PODS", (monitorKubernetesPods ? "true" : "false"))
+
+            # Check to see if monitor_kubernetes_pods is set to true with a valid setting for monitor_kubernetes_namespaces to enable scraping for specific namespaces
+            # Adding nil check here as well since checkForTypeArray returns true even if setting is nil to accomodate for other settings to be able -
+            # - to use defaults in case of nil settings
+            if monitorKubernetesPods && !monitorKubernetesPodsNamespaces.nil? && checkForTypeArray(monitorKubernetesPodsNamespaces, String)
+              new_contents = createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKubernetesPodsNamespaces, new_contents, interval, fieldPassSetting, fieldDropSetting)
+              monitorKubernetesPodsNamespacesLength = monitorKubernetesPodsNamespaces.length
+            else
+              new_contents = replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods)
+              monitorKubernetesPodsNamespacesLength = 0
+            end
+
             File.open(file_name, "w") { |file| file.puts new_contents }
             puts "config::Successfully substituted the placeholders in telegraf conf file for replicaset"
             #Set environment variables for telemetry
@@ -110,6 +174,8 @@ def populateSettingValuesFromConfigMap(parsedConfig)
               file.write("export TELEMETRY_RS_PROM_K8S_SERVICES_LENGTH=#{kubernetesServices.length}\n")
               file.write("export TELEMETRY_RS_PROM_URLS_LENGTH=#{urls.length}\n")
               file.write("export TELEMETRY_RS_PROM_MONITOR_PODS=\"#{monitorKubernetesPods}\"\n")
+              file.write("export TELEMETRY_RS_PROM_MONITOR_PODS_NS_LENGTH=\"#{monitorKubernetesPodsNamespacesLength}\"\n")
+
               # Close file after writing all environment variables
               file.close
               puts "config::Successfully created telemetry file for replicaset"

source/code/plugin/in_kube_nodes.rb

@@ -15,6 +15,7 @@ class Kube_nodeInventory_Input < Input
     @@rsPromK8sServiceCount = ENV["TELEMETRY_RS_PROM_K8S_SERVICES_LENGTH"]
     @@rsPromUrlCount = ENV["TELEMETRY_RS_PROM_URLS_LENGTH"]
     @@rsPromMonitorPods = ENV["TELEMETRY_RS_PROM_MONITOR_PODS"]
+    @@rsPromMonitorPodsNamespaceLength = ENV["TELEMETRY_RS_PROM_MONITOR_PODS_NS_LENGTH"]
 
     def initialize
       super
@@ -150,6 +151,7 @@ def enumerate
                   properties["rsPromServ"] = @@rsPromK8sServiceCount
                   properties["rsPromUrl"] = @@rsPromUrlCount
                   properties["rsPromMonPods"] = @@rsPromMonitorPods
+                  properties["rsPromMonPodsNs"] = @@rsPromMonitorPodsNamespaceLength
                 end
                 ApplicationInsightsUtility.sendMetricTelemetry("NodeCoreCapacity", capacityInfo["cpu"], properties)
                 telemetrySent = true