KubeMonAgentEvents changes to collect configuration events

installer/conf/td-agent-bit.conf

@@ -28,6 +28,7 @@
     Path /var/log/containers/omsagent*.log
     DB /var/opt/microsoft/docker-cimprov/state/omsagent-ai.db
     DB.Sync Off
+    Parser docker
     Mem_Buf_Limit 1m
     Path_Key filepath
     Skip_Long_Lines On
@@ -51,7 +52,6 @@
 [FILTER]
     Name grep
     Match oms.container.log.flbplugin.*
-    Exclude log E! [\[]inputs.prometheus[\]]
 
 [OUTPUT]
     Name                            oms

installer/datafiles/base_container.data

@@ -115,6 +115,7 @@ MAINTAINER:              'Microsoft Corporation'
 /opt/tomlparser.rb;                                                 installer/scripts/tomlparser.rb;     755; root; root
 /opt/tomlparser-prom-customconfig.rb;                               installer/scripts/tomlparser-prom-customconfig.rb;     755; root; root
 /opt/td-agent-bit-conf-customizer.rb;                               installer/scripts/td-agent-bit-conf-customizer.rb;     755; root; root
+/opt/ConfigParseErrorLogger.rb;                                     installer/scripts/ConfigParseErrorLogger.rb;           755; root; root
 
 
 

installer/scripts/ConfigParseErrorLogger.rb

@@ -0,0 +1,21 @@
+#!/usr/local/bin/ruby
+# frozen_string_literal: true
+
+class ConfigParseErrorLogger
+  require "json"
+
+  def initialize
+  end
+
+  class << self
+    def logError(message)
+      begin
+        errorMessage = "config::error::" + message
+        jsonMessage = errorMessage.to_json
+        STDERR.puts jsonMessage
+      rescue => errorStr
+        puts "Error in ConfigParserErrorLogger::logError: #{errorStr}"
+      end
+    end
+  end
+end

installer/scripts/td-agent-bit-conf-customizer.rb

@@ -1,4 +1,5 @@
 #!/usr/local/bin/ruby
+require_relative "ConfigParseErrorLogger"
 
 @td_agent_bit_conf_path = "/etc/opt/microsoft/docker-cimprov/td-agent-bit.conf"
 
@@ -40,7 +41,7 @@ def substituteFluentBitPlaceHolders
     File.open(@td_agent_bit_conf_path, "w") { |file| file.puts new_contents }
     puts "config::Successfully substituted the placeholders in td-agent-bit.conf file"
   rescue => errorStr
-    puts "td-agent-bit-config-customizer: error while substituting values: #{errorStr}"
+    ConfigParseErrorLogger.logError("td-agent-bit-config-customizer: error while substituting values in td-agent-bit.conf file: #{errorStr}")
   end
 end
 

installer/scripts/tomlparser-prom-customconfig.rb

@@ -1,6 +1,7 @@
 #!/usr/local/bin/ruby
 
 require_relative "tomlrb"
+require_relative "ConfigParseErrorLogger"
 require "fileutils"
 
 @promConfigMapMountPath = "/etc/config/settings/prometheus-data-collection-settings"
@@ -40,7 +41,7 @@ def parseConfigMap
       return nil
     end
   rescue => errorStr
-    puts "config::error::Exception while parsing toml config file for prometheus config: #{errorStr}, using defaults"
+    ConfigParseErrorLogger.logError("Exception while parsing config map for prometheus config: #{errorStr}, using defaults, please check config map for errors")
     return nil
   end
 end
@@ -66,7 +67,7 @@ def replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods)
     new_contents = new_contents.gsub("$AZMON_RS_PROM_MONITOR_PODS", ("monitor_kubernetes_pods = #{monitorKubernetesPods}"))
     new_contents = new_contents.gsub("$AZMON_RS_PROM_PLUGINS_WITH_NAMESPACE_FILTER", "")
   rescue => errorStr
-    puts "config::error::Exception while replacing default pod monitor settings: #{errorStr}"
+    puts "Exception while replacing default pod monitor settings: #{errorStr}"
   end
   return new_contents
 end
@@ -98,7 +99,7 @@ def createPrometheusPluginsWithNamespaceSetting(monitorKubernetesPods, monitorKu
     new_contents = new_contents.gsub("$AZMON_RS_PROM_PLUGINS_WITH_NAMESPACE_FILTER", pluginConfigsWithNamespaces)
     return new_contents
   rescue => errorStr
-    puts "config::error::Exception while creating prometheus input plugins to filter namespaces: #{errorStr}, using defaults"
+    puts "Exception while creating prometheus input plugins to filter namespaces: #{errorStr}, using defaults"
     replaceDefaultMonitorPodSettings(new_contents, monitorKubernetesPods)
   end
 end
@@ -181,10 +182,10 @@ def populateSettingValuesFromConfigMap(parsedConfig)
               puts "config::Successfully created telemetry file for replicaset"
             end
           else
-            puts "config::Typecheck failed for prometheus config settings for replicaset, using defaults"
+            ConfigParseErrorLogger.logError("Typecheck failed for prometheus config settings for replicaset, using defaults, please use right types for all settings")
           end # end of type check condition
         rescue => errorStr
-          puts "config::error::Exception while parsing config file for prometheus config for replicaset: #{errorStr}, using defaults"
+          ConfigParseErrorLogger.logError("Exception while parsing config file for prometheus config for replicaset: #{errorStr}, using defaults")
           setRsPromDefaults
           puts "****************End Prometheus Config Processing********************"
         end
@@ -236,16 +237,16 @@ def populateSettingValuesFromConfigMap(parsedConfig)
               puts "config::Successfully created telemetry file for daemonset"
             end
           else
-            puts "config::Typecheck failed for prometheus config settings for daemonset, using defaults"
+            ConfigParseErrorLogger.logError("Typecheck failed for prometheus config settings for daemonset, using defaults, please use right types for all settings")
           end # end of type check condition
         rescue => errorStr
-          puts "config::error::Exception while parsing config file for prometheus config for daemonset: #{errorStr}, using defaults"
+          ConfigParseErrorLogger.logError("Exception while parsing config file for prometheus config for daemonset: #{errorStr}, using defaults, please check correctness of configmap")
           puts "****************End Prometheus Config Processing********************"
         end
       end # end of controller type check
     end
   else
-    puts "config::error:: Controller undefined while processing prometheus config, using defaults"
+    ConfigParseErrorLogger.logError("Controller undefined while processing prometheus config, using defaults")
   end
 end
 
@@ -258,7 +259,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   end
 else
   if (File.file?(@promConfigMapMountPath))
-    puts "config::unsupported/missing config schema version - '#{@configSchemaVersion}' , using defaults"
+    ConfigParseErrorLogger.logError("config::unsupported/missing config schema version - '#{@configSchemaVersion}' , using defaults, please use supported version")
   else
     puts "config::No configmap mounted for prometheus custom config, using defaults"
   end

installer/scripts/tomlparser.rb

@@ -1,7 +1,8 @@
 #!/usr/local/bin/ruby
 
 require_relative "tomlrb"
-require 'json'
+require_relative "ConfigParseErrorLogger"
+require "json"
 
 @log_settings_config_map_mount_path = "/etc/config/settings/log-data-collection-settings"
 @agent_settings_config_map_mount_path = "/etc/config/settings/agent-settings"
@@ -33,7 +34,7 @@ def parseConfigMap(path)
       return nil
     end
   rescue => errorStr
-    puts "config::error::Exception while parsing toml config file: #{errorStr}, using defaults"
+    ConfigParseErrorLogger.logError("Exception while parsing config map for log collection/env variable settings: #{errorStr}, using defaults, please check config map for errors")
     @excludePath = "*_kube-system_*.log"
     return nil
   end
@@ -70,7 +71,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
         end
       end
     rescue => errorStr
-      puts "config::error::Exception while reading config settings for stdout log collection - #{errorStr}, using defaults"
+      ConfigParseErrorLogger.logError("Exception while reading config map settings for stdout log collection - #{errorStr}, using defaults, please check config map for errors")
     end
 
     #Get stderr log config settings
@@ -107,7 +108,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
         end
       end
     rescue => errorStr
-      puts "config::error:Exception while reading config settings for stderr log collection - #{errorStr}, using defaults"
+      ConfigParseErrorLogger.logError("Exception while reading config map settings for stderr log collection - #{errorStr}, using defaults, please check config map for errors")
     end
 
     #Get environment variables log config settings
@@ -117,42 +118,43 @@ def populateSettingValuesFromConfigMap(parsedConfig)
         puts "config::Using config map setting for cluster level environment variable collection"
       end
     rescue => errorStr
-      puts "config::error::Exception while reading config settings for cluster level environment variable collection - #{errorStr}, using defaults"
+      ConfigParseErrorLogger.logError("Exception while reading config map settings for cluster level environment variable collection - #{errorStr}, using defaults, please check config map for errors")
     end
   end
 
   begin
     if !parsedConfig.nil? && !parsedConfig[:agent_settings].nil? && !parsedConfig[:agent_settings][:health_model].nil? && !parsedConfig[:agent_settings][:health_model][:enabled].nil?
-        @enable_health_model = parsedConfig[:agent_settings][:health_model][:enabled]
+      @enable_health_model = parsedConfig[:agent_settings][:health_model][:enabled]
     else
-        @enable_health_model = false
+      @enable_health_model = false
     end
     puts "enable_health_model = #{@enable_health_model}"
   rescue => errorStr
-    puts "config::error:Exception while reading config settings for health_model enabled setting - #{errorStr}, using defaults"
+    ConfigParseErrorLogger.logError("Exception while reading config map settings for health_model enabled setting - #{errorStr}, using defaults, please check config map for errors")
     @enable_health_model = false
   end
 end
 
 @configSchemaVersion = ENV["AZMON_AGENT_CFG_SCHEMA_VERSION"]
 puts "****************Start Config Processing********************"
+
 if !@configSchemaVersion.nil? && !@configSchemaVersion.empty? && @configSchemaVersion.strip.casecmp("v1") == 0 #note v1 is the only supported schema version , so hardcoding it
-    configMapSettings = {}
+  configMapSettings = {}
 
-    #iterate over every *settings file and build a hash of settings
-    Dir["/etc/config/settings/*settings"].each{|file|
-        puts "Parsing File #{file}"
-        settings = parseConfigMap(file)
-        if !settings.nil?
-            configMapSettings = configMapSettings.merge(settings)
-        end
-    }
+  #iterate over every *settings file and build a hash of settings
+  Dir["/etc/config/settings/*settings"].each { |file|
+    puts "Parsing File #{file}"
+    settings = parseConfigMap(file)
+    if !settings.nil?
+      configMapSettings = configMapSettings.merge(settings)
+    end
+  }
 
   if !configMapSettings.nil?
     populateSettingValuesFromConfigMap(configMapSettings)
   end
 else
-    puts "config::unsupported/missing config schema version - '#{@configSchemaVersion}' , using defaults"
+  ConfigParseErrorLogger.logError("config::unsupported/missing config schema version - '#{@configSchemaVersion}' , using defaults, please use supported schema version")
   @excludePath = "*_kube-system_*.log"
 end
 
@@ -178,13 +180,13 @@ def populateSettingValuesFromConfigMap(parsedConfig)
   file.write("export AZMON_STDERR_EXCLUDED_NAMESPACES=#{@stderrExcludeNamespaces}\n")
   file.write("export AZMON_CLUSTER_COLLECT_ENV_VAR=#{@collectClusterEnvVariables}\n")
   file.write("export AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH=#{@excludePath}\n")
-    #health_model settings
+  #health_model settings
   file.write("export AZMON_CLUSTER_ENABLE_HEALTH_MODEL=#{@enable_health_model}\n")
   # Close file after writing all environment variables
   file.close
   puts "Both stdout & stderr log collection are turned off for namespaces: '#{@excludePath}' "
   puts "****************End Config Processing********************"
 else
-  puts "config::error::Exception while opening file for writing config environment variables"
+  puts "Exception while opening file for writing config environment variables"
   puts "****************End Config Processing********************"
 end