diff --git a/installer/conf/telegraf-rs.conf b/installer/conf/telegraf-rs.conf index ce60bfa04..3450ab88f 100644 --- a/installer/conf/telegraf-rs.conf +++ b/installer/conf/telegraf-rs.conf @@ -552,7 +552,7 @@ ## set this to `https` & most likely set the tls config. ## - prometheus.io/path: If the metrics path is not /metrics, define it with this annotation. ## - prometheus.io/port: If port is not 9102 use this annotation - monitor_kubernetes_pods = $AZMON_RS_PROM_MONITOR_PODS + $AZMON_RS_PROM_MONITOR_PODS fieldpass = $AZMON_RS_PROM_FIELDPASS fielddrop = $AZMON_RS_PROM_FIELDDROP @@ -579,6 +579,7 @@ insecure_skip_verify = true #tagexclude = ["AgentVersion","AKS_RESOURCE_ID","ACS_RESOURCE_NAME", "Region", "ClusterName", "ClusterType", "Computer", "ControllerType"] +$AZMON_RS_PROM_PLUGINS_WITH_NAMESPACE_FILTER # [[inputs.exec]] # ## Commands array # interval = "15m" diff --git a/installer/scripts/tomlparser-prom-customconfig.rb b/installer/scripts/tomlparser-prom-customconfig.rb index d9fdf1cc2..d44bf3342 100644 --- a/installer/scripts/tomlparser-prom-customconfig.rb +++ b/installer/scripts/tomlparser-prom-customconfig.rb @@ -18,6 +18,14 @@ @defaultRsK8sServices = [] @defaultRsMonitorPods = false +#Configurations to be used for the auto-generated input prometheus plugins for namespace filtering +@metricVersion = 2 +@urlTag = "scrapeUrl" +@bearerToken = "/var/run/secrets/kubernetes.io/serviceaccount/token" +@responseTimeout = "15s" +@tlsCa = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" +@insecureSkipVerify = true + # Use parser to parse the configmap toml file to a ruby structure def parseConfigMap begin @@ -53,6 +61,48 @@ def checkForType(variable, varType) end end +def replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods) + begin + new_contents = new_contents.gsub("$AZMON_RS_PROM_MONITOR_PODS", ("monitor_kubernetes_pods = #{monitorKubernetesPods}")) + new_contents = new_contents.gsub("$AZMON_RS_PROM_PLUGINS_WITH_NAMESPACE_FILTER", "") + rescue => errorStr + puts "config::error::Exception while replacing default pod monitor settings: #{errorStr}" + end + return new_contents +end + +def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKubernetesPodsNamespaces, new_contents, interval, fieldPassSetting, fieldDropSetting) + begin + new_contents = new_contents.gsub("$AZMON_RS_PROM_MONITOR_PODS", "# Commenting this out since new plugins will be created per namespace\n # $AZMON_RS_PROM_MONITOR_PODS") + pluginConfigsWithNamespaces = "" + monitorKubernetesPodsNamespaces.each do |namespace| + if !namespace.nil? + #Stripping namespaces to remove leading and trailing whitespaces + namespace.strip! + if namespace.length > 0 + pluginConfigsWithNamespaces += "\n[[inputs.prometheus]] + interval = \"#{interval}\" + monitor_kubernetes_pods = true + monitor_kubernetes_pods_namespace = \"#{namespace}\" + fieldpass = #{fieldPassSetting} + fielddrop = #{fieldDropSetting} + metric_version = #{@metricVersion} + url_tag = \"#{@urlTag}\" + bearer_token = \"#{@bearerToken}\" + response_timeout = \"#{@responseTimeout}\" + tls_ca = \"#{@tlsCa}\" + insecure_skip_verify = #{@insecureSkipVerify}\n" + end + end + end + new_contents = new_contents.gsub("$AZMON_RS_PROM_PLUGINS_WITH_NAMESPACE_FILTER", pluginConfigsWithNamespaces) + return new_contents + rescue => errorStr + puts "config::error::Exception while creating prometheus input plugins to filter namespaces: #{errorStr}, using defaults" + replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods) + end +end + # Use the ruby structure created after config parsing to set the right values to be used as environment variables def populateSettingValuesFromConfigMap(parsedConfig) # Checking to see if this is the daemonset or replicaset to parse config accordingly @@ -68,6 +118,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) urls = parsedConfig[:prometheus_data_collection_settings][:cluster][:urls] kubernetesServices = parsedConfig[:prometheus_data_collection_settings][:cluster][:kubernetes_services] monitorKubernetesPods = parsedConfig[:prometheus_data_collection_settings][:cluster][:monitor_kubernetes_pods] + monitorKubernetesPodsNamespaces = parsedConfig[:prometheus_data_collection_settings][:cluster][:monitor_kubernetes_pods_namespaces] # Check for the right datattypes to enforce right setting values if checkForType(interval, String) && @@ -75,7 +126,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) checkForTypeArray(fieldDrop, String) && checkForTypeArray(kubernetesServices, String) && checkForTypeArray(urls, String) && - !monitorKubernetesPods.nil? && (!!monitorKubernetesPods == monitorKubernetesPods) #Checking for Boolean type, since 'Boolean' is not defined as a type in ruby + (monitorKubernetesPods.nil? || (!monitorKubernetesPods.nil? && (!!monitorKubernetesPods == monitorKubernetesPods))) #Checking for Boolean type, since 'Boolean' is not defined as a type in ruby puts "config::Successfully passed typecheck for config settings for replicaset" #if setting is nil assign default values interval = (interval.nil?) ? @defaultRsInterval : interval @@ -83,7 +134,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) fieldDrop = (fieldDrop.nil?) ? @defaultRsFieldDrop : fieldDrop kubernetesServices = (kubernetesServices.nil?) ? @defaultRsK8sServices : kubernetesServices urls = (urls.nil?) ? @defaultRsPromUrls : urls - monitorKubernetesPods = (kubernetesServices.nil?) ? @defaultRsMonitorPods : monitorKubernetesPods + monitorKubernetesPods = (monitorKubernetesPods.nil?) ? @defaultRsMonitorPods : monitorKubernetesPods file_name = "/opt/telegraf-test-rs.conf" # Copy the telegraf config file to a temp file to run telegraf in test mode with this config @@ -93,11 +144,24 @@ def populateSettingValuesFromConfigMap(parsedConfig) #Replace the placeholder config values with values from custom config text = File.read(file_name) new_contents = text.gsub("$AZMON_RS_PROM_INTERVAL", interval) - new_contents = new_contents.gsub("$AZMON_RS_PROM_FIELDPASS", ((fieldPass.length > 0) ? ("[\"" + fieldPass.join("\",\"") + "\"]") : "[]")) - new_contents = new_contents.gsub("$AZMON_RS_PROM_FIELDDROP", ((fieldDrop.length > 0) ? ("[\"" + fieldDrop.join("\",\"") + "\"]") : "[]")) + fieldPassSetting = (fieldPass.length > 0) ? ("[\"" + fieldPass.join("\",\"") + "\"]") : "[]" + new_contents = new_contents.gsub("$AZMON_RS_PROM_FIELDPASS", fieldPassSetting) + fieldDropSetting = (fieldDrop.length > 0) ? ("[\"" + fieldDrop.join("\",\"") + "\"]") : "[]" + new_contents = new_contents.gsub("$AZMON_RS_PROM_FIELDDROP", fieldDropSetting) new_contents = new_contents.gsub("$AZMON_RS_PROM_URLS", ((urls.length > 0) ? ("[\"" + urls.join("\",\"") + "\"]") : "[]")) new_contents = new_contents.gsub("$AZMON_RS_PROM_K8S_SERVICES", ((kubernetesServices.length > 0) ? ("[\"" + kubernetesServices.join("\",\"") + "\"]") : "[]")) - new_contents = new_contents.gsub("$AZMON_RS_PROM_MONITOR_PODS", (monitorKubernetesPods ? "true" : "false")) + + # Check to see if monitor_kubernetes_pods is set to true with a valid setting for monitor_kubernetes_namespaces to enable scraping for specific namespaces + # Adding nil check here as well since checkForTypeArray returns true even if setting is nil to accomodate for other settings to be able - + # - to use defaults in case of nil settings + if monitorKubernetesPods && !monitorKubernetesPodsNamespaces.nil? && checkForTypeArray(monitorKubernetesPodsNamespaces, String) + new_contents = createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKubernetesPodsNamespaces, new_contents, interval, fieldPassSetting, fieldDropSetting) + monitorKubernetesPodsNamespacesLength = monitorKubernetesPodsNamespaces.length + else + new_contents = replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods) + monitorKubernetesPodsNamespacesLength = 0 + end + File.open(file_name, "w") { |file| file.puts new_contents } puts "config::Successfully substituted the placeholders in telegraf conf file for replicaset" #Set environment variables for telemetry @@ -110,6 +174,8 @@ def populateSettingValuesFromConfigMap(parsedConfig) file.write("export TELEMETRY_RS_PROM_K8S_SERVICES_LENGTH=#{kubernetesServices.length}\n") file.write("export TELEMETRY_RS_PROM_URLS_LENGTH=#{urls.length}\n") file.write("export TELEMETRY_RS_PROM_MONITOR_PODS=\"#{monitorKubernetesPods}\"\n") + file.write("export TELEMETRY_RS_PROM_MONITOR_PODS_NS_LENGTH=\"#{monitorKubernetesPodsNamespacesLength}\"\n") + # Close file after writing all environment variables file.close puts "config::Successfully created telemetry file for replicaset" diff --git a/source/code/plugin/in_kube_nodes.rb b/source/code/plugin/in_kube_nodes.rb index 24ab51d4c..7249957ab 100644 --- a/source/code/plugin/in_kube_nodes.rb +++ b/source/code/plugin/in_kube_nodes.rb @@ -15,6 +15,7 @@ class Kube_nodeInventory_Input < Input @@rsPromK8sServiceCount = ENV["TELEMETRY_RS_PROM_K8S_SERVICES_LENGTH"] @@rsPromUrlCount = ENV["TELEMETRY_RS_PROM_URLS_LENGTH"] @@rsPromMonitorPods = ENV["TELEMETRY_RS_PROM_MONITOR_PODS"] + @@rsPromMonitorPodsNamespaceLength = ENV["TELEMETRY_RS_PROM_MONITOR_PODS_NS_LENGTH"] def initialize super @@ -150,6 +151,7 @@ def enumerate properties["rsPromServ"] = @@rsPromK8sServiceCount properties["rsPromUrl"] = @@rsPromUrlCount properties["rsPromMonPods"] = @@rsPromMonitorPods + properties["rsPromMonPodsNs"] = @@rsPromMonitorPodsNamespaceLength end ApplicationInsightsUtility.sendMetricTelemetry("NodeCoreCapacity", capacityInfo["cpu"], properties) telemetrySent = true