From 13404a03ba7001971d987961ff8300b2d302d7ea Mon Sep 17 00:00:00 2001 From: Rashmi Chandrashekar Date: Mon, 24 May 2021 17:25:33 -0700 Subject: [PATCH 01/12] changes --- .../scripts/tomlparser-agent-config.rb | 18 +++++++++++++----- kubernetes/container-azm-ms-agentconfig.yaml | 8 ++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/build/linux/installer/scripts/tomlparser-agent-config.rb b/build/linux/installer/scripts/tomlparser-agent-config.rb index e587909e5..8dfaf19e1 100644 --- a/build/linux/installer/scripts/tomlparser-agent-config.rb +++ b/build/linux/installer/scripts/tomlparser-agent-config.rb @@ -13,6 +13,8 @@ @configMapMountPath = "/etc/config/settings/agent-settings" @configSchemaVersion = "" @enable_health_model = false +@containerType = ENV["CONTAINER_TYPE"] +@promSideCar = "prometheussidecar" # 250 Node items (15KB per node) account to approximately 4MB @nodesChunkSize = 250 @@ -60,7 +62,6 @@ @fbitTailBufferChunkSizeMBs = 0 @fbitTailBufferMaxSizeMBs = 0 - def is_number?(value) true if Integer(value) rescue false end @@ -153,7 +154,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) end fbitTailBufferMaxSizeMBs = fbit_config[:tail_buf_maxsize_megabytes] - if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0 + if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0 if fbitTailBufferMaxSizeMBs.to_i >= @fbitTailBufferChunkSizeMBs @fbitTailBufferMaxSizeMBs = fbitTailBufferMaxSizeMBs.to_i puts "Using config map value: tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs}" @@ -164,10 +165,17 @@ def populateSettingValuesFromConfigMap(parsedConfig) end end # in scenario - tail_buf_chunksize_megabytes provided but not tail_buf_maxsize_megabytes to prevent fbit crash - if @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0 + if @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0 @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs puts "config::warn: since tail_buf_maxsize_megabytes not provided hence using tail_buf_maxsize_megabytes=#{@fbitTailBufferMaxSizeMBs} which is same as the value of tail_buf_chunksize_megabytes" - end + end + end + + # fbit config settings + if (!@containerType.nil? && @containerType.casecmp(@promSideCar) == 0) + prom_fbit_config = parsedConfig[:agent_settings][:prometheus_fbit_settings] + if !prom_fbit_config.nil? + end end end rescue => errorStr @@ -211,7 +219,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) end if @fbitTailBufferMaxSizeMBs > 0 file.write("export FBIT_TAIL_BUFFER_MAX_SIZE=#{@fbitTailBufferMaxSizeMBs}\n") - end + end # Close file after writing all environment variables file.close else diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index 543f270c1..c1d4a9fc4 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -135,6 +135,14 @@ data: [integrations.azure_network_policy_manager] collect_basic_metrics = false collect_advanced_metrics = false + + agent-settings: |- + # prometheus scrape fluent bit settings for high scale + [agent_settings.prometheus_fbit_settings] + tcp_listener_chunk_size = 10 + tcp_listener_buffer_size = 10 + tcp_listener_mem_buf_limit = 200 + metadata: name: container-azm-ms-agentconfig namespace: kube-system From 57b7d84bea575f80f0ba9ea261e301dcd4adbc2e Mon Sep 17 00:00:00 2001 From: Rashmi Chandrashekar Date: Wed, 26 May 2021 10:27:47 -0700 Subject: [PATCH 02/12] changes --- .../scripts/tomlparser-agent-config.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/build/linux/installer/scripts/tomlparser-agent-config.rb b/build/linux/installer/scripts/tomlparser-agent-config.rb index 8dfaf19e1..0c6eaee42 100644 --- a/build/linux/installer/scripts/tomlparser-agent-config.rb +++ b/build/linux/installer/scripts/tomlparser-agent-config.rb @@ -62,6 +62,10 @@ @fbitTailBufferChunkSizeMBs = 0 @fbitTailBufferMaxSizeMBs = 0 +@promFbitChunkSize = 10 +@promFbitBufferSize = 10 +@promFbitMemBufLimit = 200 + def is_number?(value) true if Integer(value) rescue false end @@ -175,6 +179,21 @@ def populateSettingValuesFromConfigMap(parsedConfig) if (!@containerType.nil? && @containerType.casecmp(@promSideCar) == 0) prom_fbit_config = parsedConfig[:agent_settings][:prometheus_fbit_settings] if !prom_fbit_config.nil? + chunk_size = prom_fbit_config[:tcp_listener_chunk_size] + if !chunk_size.nil? && is_number?(chunk_size) && chunk_size.to_i > 0 + @promFbitChunkSize = chunk_size.to_i + end + buffer_size = prom_fbit_config[:tcp_listener_buffer_size] + if !buffer_size.nil? && is_number?(buffer_size) && buffer_size.to_i > 0 + @promFbitBufferSize = buffer_size.to_i + if @promFbitBufferSize < @promFbitChunkSize + @promFbitBufferSize = @promFbitChunkSize + end + end + mem_buf_limit = prom_fbit_config[:tcp_listener_mem_buf_limit] + if !mem_buf_limit.nil? && is_number?(mem_buf_limit) && mem_buf_limit.to_i > 0 + @promFbitMemBufLimit = mem_buf_limit.to_i + end end end end From 279073cf3de6575c8e49bb43053910bbbc43f9b2 Mon Sep 17 00:00:00 2001 From: Rashmi Chandrashekar Date: Wed, 26 May 2021 17:31:47 -0700 Subject: [PATCH 03/12] changes --- kubernetes/container-azm-ms-agentconfig.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index c1d4a9fc4..3b47e0262 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -137,7 +137,8 @@ data: collect_advanced_metrics = false agent-settings: |- - # prometheus scrape fluent bit settings for high scale + # prometheus scrape fluent bit settings for high scale + # buffer size should be greater than or equal to chunk size else we set it to chunk size. [agent_settings.prometheus_fbit_settings] tcp_listener_chunk_size = 10 tcp_listener_buffer_size = 10 From 434370da268d443062413c4fd9db17bbeeea85df Mon Sep 17 00:00:00 2001 From: Rashmi Chandrashekar Date: Mon, 31 May 2021 15:27:28 -0700 Subject: [PATCH 04/12] changes --- .../scripts/tomlparser-prom-agent-config.rb | 98 +++++++++++++++++++ .../conf/td-agent-bit-prom-side-car.conf | 6 +- .../installer/datafiles/base_container.data | 1 + .../linux/defaultpromenvvariables-sidecar | 3 + kubernetes/linux/main.sh | 15 +++ 5 files changed, 120 insertions(+), 3 deletions(-) create mode 100644 build/common/installer/scripts/tomlparser-prom-agent-config.rb diff --git a/build/common/installer/scripts/tomlparser-prom-agent-config.rb b/build/common/installer/scripts/tomlparser-prom-agent-config.rb new file mode 100644 index 000000000..424d0ca37 --- /dev/null +++ b/build/common/installer/scripts/tomlparser-prom-agent-config.rb @@ -0,0 +1,98 @@ +#!/usr/local/bin/ruby + +#this should be require relative in Linux and require in windows, since it is a gem install on windows +@os_type = ENV["OS_TYPE"] +if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 + require "tomlrb" +else + require_relative "tomlrb" +end + +require_relative "ConfigParseErrorLogger" + +@configMapMountPath = "/etc/config/settings/agent-settings" +@configSchemaVersion = "" + +@promFbitChunkSize = 10 +@promFbitBufferSize = 10 +@promFbitMemBufLimit = 200 + +def is_number?(value) + true if Integer(value) rescue false +end + +# Use parser to parse the configmap toml file to a ruby structure +def parseConfigMap + begin + # Check to see if config map is created + if (File.file?(@configMapMountPath)) + puts "config::configmap container-azm-ms-agentconfig for sidecar agent settings mounted, parsing values" + parsedConfig = Tomlrb.load_file(@configMapMountPath, symbolize_keys: true) + puts "config::Successfully parsed mounted config map" + return parsedConfig + else + puts "config::configmap container-azm-ms-agentconfig for sidecar agent settings not mounted, using defaults" + return nil + end + rescue => errorStr + ConfigParseErrorLogger.logError("Exception while parsing config map for sidecar agent settings : #{errorStr}, using defaults, please check config map for errors") + return nil + end +end + +# Use the ruby structure created after config parsing to set the right values to be used as environment variables +def populateSettingValuesFromConfigMap(parsedConfig) + begin + if !parsedConfig.nil? && !parsedConfig[:agent_settings].nil? + # fbit config settings + prom_fbit_config = parsedConfig[:agent_settings][:prometheus_fbit_settings] + if !prom_fbit_config.nil? + chunk_size = prom_fbit_config[:tcp_listener_chunk_size] + if !chunk_size.nil? && is_number?(chunk_size) && chunk_size.to_i > 0 + @promFbitChunkSize = chunk_size.to_i + end + buffer_size = prom_fbit_config[:tcp_listener_buffer_size] + if !buffer_size.nil? && is_number?(buffer_size) && buffer_size.to_i > 0 + @promFbitBufferSize = buffer_size.to_i + if @promFbitBufferSize < @promFbitChunkSize + @promFbitBufferSize = @promFbitChunkSize + end + end + mem_buf_limit = prom_fbit_config[:tcp_listener_mem_buf_limit] + if !mem_buf_limit.nil? && is_number?(mem_buf_limit) && mem_buf_limit.to_i > 0 + @promFbitMemBufLimit = mem_buf_limit.to_i + end + end + end + rescue => errorStr + puts "config::error:Exception while reading config settings for sidecar agent configuration setting - #{errorStr}, using defaults" + end +end + +@configSchemaVersion = ENV["AZMON_AGENT_CFG_SCHEMA_VERSION"] +puts "****************Start Sidecar Agent Config Processing********************" +if !@configSchemaVersion.nil? && !@configSchemaVersion.empty? && @configSchemaVersion.strip.casecmp("v1") == 0 #note v1 is the only supported schema version , so hardcoding it + configMapSettings = parseConfigMap + if !configMapSettings.nil? + populateSettingValuesFromConfigMap(configMapSettings) + end +else + if (File.file?(@configMapMountPath)) + ConfigParseErrorLogger.logError("config::unsupported/missing config schema version - '#{@configSchemaVersion}' , using defaults, please use supported schema version") + end + @enable_health_model = false +end + +# Write the settings to file, so that they can be set as environment variables +file = File.open("side_car_fbit_config_env_var", "w") + +if !file.nil? + file.write("export AZMON_SIDECAR_FBIT_CHUNK_SIZE=#{@promFbitChunkSize + "m"}\n") + file.write("export AZMON_SIDECAR_FBIT_BUFFER_SIZE=#{@promFbitBufferSize + "m"}\n") + file.write("export AZMON_SIDECAR_FBIT_MEM_BUF_LIMIT=#{@promFbitMemBufLimit + "m"}\n") + # Close file after writing all environment variables + file.close +else + puts "Exception while opening file for writing config environment variables" + puts "****************End Sidecar Agent Config Processing********************" +end diff --git a/build/linux/installer/conf/td-agent-bit-prom-side-car.conf b/build/linux/installer/conf/td-agent-bit-prom-side-car.conf index 8a69f7995..2c85a4200 100644 --- a/build/linux/installer/conf/td-agent-bit-prom-side-car.conf +++ b/build/linux/installer/conf/td-agent-bit-prom-side-car.conf @@ -29,9 +29,9 @@ Tag oms.container.perf.telegraf.* Listen 0.0.0.0 Port 25229 - Chunk_Size 10m - Buffer_Size 10m - Mem_Buf_Limit 200m + Chunk_Size ${AZMON_SIDECAR_FBIT_CHUNK_SIZE} + Buffer_Size ${AZMON_SIDECAR_FBIT_BUFFER_SIZE} + Mem_Buf_Limit ${AZMON_SIDECAR_FBIT_MEM_BUF_LIMIT} [OUTPUT] Name oms diff --git a/build/linux/installer/datafiles/base_container.data b/build/linux/installer/datafiles/base_container.data index b9f889dba..da9fdf241 100644 --- a/build/linux/installer/datafiles/base_container.data +++ b/build/linux/installer/datafiles/base_container.data @@ -42,6 +42,7 @@ MAINTAINER: 'Microsoft Corporation' /opt/microsoft/docker-cimprov/bin/TelegrafTCPErrorTelemetry.sh; build/linux/installer/scripts/TelegrafTCPErrorTelemetry.sh; 755; root; root /opt/livenessprobe.sh; build/linux/installer/scripts/livenessprobe.sh; 755; root; root /opt/tomlparser-prom-customconfig.rb; build/common/installer/scripts/tomlparser-prom-customconfig.rb; 755; root; root +/opt/tomlparser-prom-agent-config.rb; build/common/installer/scripts/tomlparser-prom-agent-config.rb; 755; root; root /opt/tomlparser-mdm-metrics-config.rb; build/linux/installer/scripts/tomlparser-mdm-metrics-config.rb; 755; root; root /opt/tomlparser-metric-collection-config.rb; build/linux/installer/scripts/tomlparser-metric-collection-config.rb; 755; root; root diff --git a/kubernetes/linux/defaultpromenvvariables-sidecar b/kubernetes/linux/defaultpromenvvariables-sidecar index 3301488d8..68388f88e 100644 --- a/kubernetes/linux/defaultpromenvvariables-sidecar +++ b/kubernetes/linux/defaultpromenvvariables-sidecar @@ -7,3 +7,6 @@ export AZMON_TELEGRAF_CUSTOM_PROM_PLUGINS_WITH_NAMESPACE_FILTER="" export AZMON_TELEGRAF_OSM_PROM_PLUGINS="" export AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_LABEL_SELECTOR="kubernetes_label_selector = ''" export AZMON_TELEGRAF_CUSTOM_PROM_KUBERNETES_FIELD_SELECTOR="kubernetes_field_selector = ''" +export AZMON_SIDECAR_FBIT_CHUNK_SIZE="10m" +export AZMON_SIDECAR_FBIT_BUFFER_SIZE="10m" +export AZMON_SIDECAR_FBIT_MEM_BUF_LIMIT="200m" diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index b21ed6b96..ff8572ca8 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -306,6 +306,21 @@ if [ -e "telemetry_prom_config_env_var" ]; then source telemetry_prom_config_env_var fi +#Parse sidecar agent settings for custom configuration +if [ ! -e "/etc/config/kube.conf" ]; then + if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then + #Parse the agent configmap to create a file with new custom settings. + /usr/bin/ruby2.6 tomlparser-prom-agent-config.rb + #Sourcing config environment variable file if it exists + if [ -e "side_car_fbit_config_env_var" ]; then + cat side_car_fbit_config_env_var | while read line; do + echo $line >> ~/.bashrc + done + source side_car_fbit_config_env_var + fi + fi +fi + #Parse the configmap to set the right environment variables for MDM metrics configuration for Alerting. if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then From 0f89133c3d2624017b86ec3e0a82d09f82a93f65 Mon Sep 17 00:00:00 2001 From: Rashmi Chandrashekar Date: Tue, 1 Jun 2021 16:22:29 -0700 Subject: [PATCH 05/12] telemetry changes --- source/plugins/go/src/telemetry.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/plugins/go/src/telemetry.go b/source/plugins/go/src/telemetry.go index 4750b4624..97a1bc48c 100644 --- a/source/plugins/go/src/telemetry.go +++ b/source/plugins/go/src/telemetry.go @@ -161,6 +161,12 @@ func SendContainerLogPluginMetrics(telemetryPushIntervalProperty string) { telemetryDimensions["OsmNamespaceCount"] = strconv.Itoa(osmNamespaceCount) } + telemetryDimensions["PromFbitChunkSize"] = os.Getenv("AZMON_SIDECAR_FBIT_CHUNK_SIZE") + telemetryDimensions["PromFbitBufferSize"] = os.Getenv("AZMON_SIDECAR_FBIT_BUFFER_SIZE") + telemetryDimensions["PromFbitMemBufLimit"] = os.Getenv("AZMON_SIDECAR_FBIT_MEM_BUF_LIMIT") + // Delete the line below + telemetryDimensions["RashmiPromFbitMemBufLimit"] = os.Getenv("RASHMI_AZMON_SIDECAR_FBIT_MEM_BUF_LIMIT") + SendEvent(eventNameCustomPrometheusSidecarHeartbeat, telemetryDimensions) } else { From ab352cd7df18ac4f181b5d9968159d3437763770 Mon Sep 17 00:00:00 2001 From: Rashmi Chandrashekar Date: Tue, 1 Jun 2021 17:07:37 -0700 Subject: [PATCH 06/12] fixing merge issue --- build/linux/installer/datafiles/base_container.data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/linux/installer/datafiles/base_container.data b/build/linux/installer/datafiles/base_container.data index da9fdf241..88c790be3 100644 --- a/build/linux/installer/datafiles/base_container.data +++ b/build/linux/installer/datafiles/base_container.data @@ -43,7 +43,7 @@ MAINTAINER: 'Microsoft Corporation' /opt/livenessprobe.sh; build/linux/installer/scripts/livenessprobe.sh; 755; root; root /opt/tomlparser-prom-customconfig.rb; build/common/installer/scripts/tomlparser-prom-customconfig.rb; 755; root; root /opt/tomlparser-prom-agent-config.rb; build/common/installer/scripts/tomlparser-prom-agent-config.rb; 755; root; root -/opt/tomlparser-mdm-metrics-config.rb; build/linux/installer/scripts/tomlparser-mdm-metrics-config.rb; 755; root; root +/opt/tomlparser-mdm-metrics-config.rb; build/common/installer/scripts/tomlparser-mdm-metrics-config.rb; 755; root; root /opt/tomlparser-metric-collection-config.rb; build/linux/installer/scripts/tomlparser-metric-collection-config.rb; 755; root; root From 80655f89fd8fb69742b8db3c3c8a15da4d94612b Mon Sep 17 00:00:00 2001 From: Rashmi Chandrashekar Date: Tue, 1 Jun 2021 17:22:49 -0700 Subject: [PATCH 07/12] convert to string --- .../installer/scripts/tomlparser-prom-agent-config.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build/common/installer/scripts/tomlparser-prom-agent-config.rb b/build/common/installer/scripts/tomlparser-prom-agent-config.rb index 424d0ca37..e040793db 100644 --- a/build/common/installer/scripts/tomlparser-prom-agent-config.rb +++ b/build/common/installer/scripts/tomlparser-prom-agent-config.rb @@ -87,9 +87,9 @@ def populateSettingValuesFromConfigMap(parsedConfig) file = File.open("side_car_fbit_config_env_var", "w") if !file.nil? - file.write("export AZMON_SIDECAR_FBIT_CHUNK_SIZE=#{@promFbitChunkSize + "m"}\n") - file.write("export AZMON_SIDECAR_FBIT_BUFFER_SIZE=#{@promFbitBufferSize + "m"}\n") - file.write("export AZMON_SIDECAR_FBIT_MEM_BUF_LIMIT=#{@promFbitMemBufLimit + "m"}\n") + file.write("export AZMON_SIDECAR_FBIT_CHUNK_SIZE=#{@promFbitChunkSize.to_s + "m"}\n") + file.write("export AZMON_SIDECAR_FBIT_BUFFER_SIZE=#{@promFbitBufferSize.to_s + "m"}\n") + file.write("export AZMON_SIDECAR_FBIT_MEM_BUF_LIMIT=#{@promFbitMemBufLimit.to_s + "m"}\n") # Close file after writing all environment variables file.close else From aaf72cd1a577904b3ddc564de3546e7d248df5da Mon Sep 17 00:00:00 2001 From: Rashmi Chandrashekar Date: Tue, 1 Jun 2021 17:56:18 -0700 Subject: [PATCH 08/12] adding comments --- .../common/installer/scripts/tomlparser-prom-agent-config.rb | 4 ++++ source/plugins/go/src/telemetry.go | 2 -- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/build/common/installer/scripts/tomlparser-prom-agent-config.rb b/build/common/installer/scripts/tomlparser-prom-agent-config.rb index e040793db..be9d08e59 100644 --- a/build/common/installer/scripts/tomlparser-prom-agent-config.rb +++ b/build/common/installer/scripts/tomlparser-prom-agent-config.rb @@ -50,17 +50,21 @@ def populateSettingValuesFromConfigMap(parsedConfig) chunk_size = prom_fbit_config[:tcp_listener_chunk_size] if !chunk_size.nil? && is_number?(chunk_size) && chunk_size.to_i > 0 @promFbitChunkSize = chunk_size.to_i + puts "Using config map value: AZMON_SIDECAR_FBIT_CHUNK_SIZE = #{@promFbitChunkSize.to_s + "m"}" end buffer_size = prom_fbit_config[:tcp_listener_buffer_size] if !buffer_size.nil? && is_number?(buffer_size) && buffer_size.to_i > 0 @promFbitBufferSize = buffer_size.to_i + puts "Using config map value: AZMON_SIDECAR_FBIT_BUFFER_SIZE = #{@promFbitBufferSize.to_s + "m"}" if @promFbitBufferSize < @promFbitChunkSize @promFbitBufferSize = @promFbitChunkSize + puts "Setting Fbit buffer size equal to chunk size since it is set to less than chunk size - AZMON_SIDECAR_FBIT_BUFFER_SIZE = #{@promFbitBufferSize.to_s + "m"}" end end mem_buf_limit = prom_fbit_config[:tcp_listener_mem_buf_limit] if !mem_buf_limit.nil? && is_number?(mem_buf_limit) && mem_buf_limit.to_i > 0 @promFbitMemBufLimit = mem_buf_limit.to_i + puts "Using config map value: AZMON_SIDECAR_FBIT_MEM_BUF_LIMIT = #{@promFbitMemBufLimit.to_s + "m"}" end end end diff --git a/source/plugins/go/src/telemetry.go b/source/plugins/go/src/telemetry.go index 97a1bc48c..debe003e4 100644 --- a/source/plugins/go/src/telemetry.go +++ b/source/plugins/go/src/telemetry.go @@ -164,8 +164,6 @@ func SendContainerLogPluginMetrics(telemetryPushIntervalProperty string) { telemetryDimensions["PromFbitChunkSize"] = os.Getenv("AZMON_SIDECAR_FBIT_CHUNK_SIZE") telemetryDimensions["PromFbitBufferSize"] = os.Getenv("AZMON_SIDECAR_FBIT_BUFFER_SIZE") telemetryDimensions["PromFbitMemBufLimit"] = os.Getenv("AZMON_SIDECAR_FBIT_MEM_BUF_LIMIT") - // Delete the line below - telemetryDimensions["RashmiPromFbitMemBufLimit"] = os.Getenv("RASHMI_AZMON_SIDECAR_FBIT_MEM_BUF_LIMIT") SendEvent(eventNameCustomPrometheusSidecarHeartbeat, telemetryDimensions) From 97c9a1ed6942e0079d43bf9b4ce1b8bd19776585 Mon Sep 17 00:00:00 2001 From: Rashmi Chandrashekar Date: Wed, 2 Jun 2021 10:25:14 -0700 Subject: [PATCH 09/12] reverting changes --- .../scripts/tomlparser-agent-config.rb | 35 +++---------------- 1 file changed, 4 insertions(+), 31 deletions(-) diff --git a/build/linux/installer/scripts/tomlparser-agent-config.rb b/build/linux/installer/scripts/tomlparser-agent-config.rb index 0c6eaee42..e587909e5 100644 --- a/build/linux/installer/scripts/tomlparser-agent-config.rb +++ b/build/linux/installer/scripts/tomlparser-agent-config.rb @@ -13,8 +13,6 @@ @configMapMountPath = "/etc/config/settings/agent-settings" @configSchemaVersion = "" @enable_health_model = false -@containerType = ENV["CONTAINER_TYPE"] -@promSideCar = "prometheussidecar" # 250 Node items (15KB per node) account to approximately 4MB @nodesChunkSize = 250 @@ -62,9 +60,6 @@ @fbitTailBufferChunkSizeMBs = 0 @fbitTailBufferMaxSizeMBs = 0 -@promFbitChunkSize = 10 -@promFbitBufferSize = 10 -@promFbitMemBufLimit = 200 def is_number?(value) true if Integer(value) rescue false @@ -158,7 +153,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) end fbitTailBufferMaxSizeMBs = fbit_config[:tail_buf_maxsize_megabytes] - if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0 + if !fbitTailBufferMaxSizeMBs.nil? && is_number?(fbitTailBufferMaxSizeMBs) && fbitTailBufferMaxSizeMBs.to_i > 0 if fbitTailBufferMaxSizeMBs.to_i >= @fbitTailBufferChunkSizeMBs @fbitTailBufferMaxSizeMBs = fbitTailBufferMaxSizeMBs.to_i puts "Using config map value: tail_buf_maxsize_megabytes = #{@fbitTailBufferMaxSizeMBs}" @@ -169,32 +164,10 @@ def populateSettingValuesFromConfigMap(parsedConfig) end end # in scenario - tail_buf_chunksize_megabytes provided but not tail_buf_maxsize_megabytes to prevent fbit crash - if @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0 + if @fbitTailBufferChunkSizeMBs > 0 && @fbitTailBufferMaxSizeMBs == 0 @fbitTailBufferMaxSizeMBs = @fbitTailBufferChunkSizeMBs puts "config::warn: since tail_buf_maxsize_megabytes not provided hence using tail_buf_maxsize_megabytes=#{@fbitTailBufferMaxSizeMBs} which is same as the value of tail_buf_chunksize_megabytes" - end - end - - # fbit config settings - if (!@containerType.nil? && @containerType.casecmp(@promSideCar) == 0) - prom_fbit_config = parsedConfig[:agent_settings][:prometheus_fbit_settings] - if !prom_fbit_config.nil? - chunk_size = prom_fbit_config[:tcp_listener_chunk_size] - if !chunk_size.nil? && is_number?(chunk_size) && chunk_size.to_i > 0 - @promFbitChunkSize = chunk_size.to_i - end - buffer_size = prom_fbit_config[:tcp_listener_buffer_size] - if !buffer_size.nil? && is_number?(buffer_size) && buffer_size.to_i > 0 - @promFbitBufferSize = buffer_size.to_i - if @promFbitBufferSize < @promFbitChunkSize - @promFbitBufferSize = @promFbitChunkSize - end - end - mem_buf_limit = prom_fbit_config[:tcp_listener_mem_buf_limit] - if !mem_buf_limit.nil? && is_number?(mem_buf_limit) && mem_buf_limit.to_i > 0 - @promFbitMemBufLimit = mem_buf_limit.to_i - end - end + end end end rescue => errorStr @@ -238,7 +211,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) end if @fbitTailBufferMaxSizeMBs > 0 file.write("export FBIT_TAIL_BUFFER_MAX_SIZE=#{@fbitTailBufferMaxSizeMBs}\n") - end + end # Close file after writing all environment variables file.close else From 69f0a7249cdd6559a27ffc5da117900f09ecf949 Mon Sep 17 00:00:00 2001 From: Rashmi Chandrashekar Date: Mon, 14 Jun 2021 11:57:09 -0700 Subject: [PATCH 10/12] doc update --- Documentation/AgentSettings/ReadMe.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 Documentation/AgentSettings/ReadMe.md diff --git a/Documentation/AgentSettings/ReadMe.md b/Documentation/AgentSettings/ReadMe.md new file mode 100644 index 000000000..cb6dcb7f6 --- /dev/null +++ b/Documentation/AgentSettings/ReadMe.md @@ -0,0 +1,26 @@ +## Configurable agent settings for high scale prometheus metric scraping using pod annotations with prometheus sidecar. + +Container Insights agent runs native prometheus telegraf plugin to scrape prometheus metrics using pod annotations. +The metrics scraped from the telegraf plugin are sent to the fluent bit tcp listener. +In order to support higher volumes of prometheus metrics scraping some of the tcp listener settings can be tuned. +[Fluent Bit TCP listener](https://docs.fluentbit.io/manual/pipeline/inputs/tcp) + +Chunk Size - This can be increased to process bigger chunks of data. + +Buffer Size - This should be greater than or equal to the chunk size. + +Mem Buf Limit - This can be increased to increase the buffer size. But the memory limit on the sidecar also needs to be increased accordingly. +Note that this can only be achieved using helm chart today. + + +* Note - The LA ingestion team also states that higher chunk sizes might not necessarily mean higher throughput since there are pipeline limitations. + +``` + agent-settings: |- + # prometheus scrape fluent bit settings for high scale + # buffer size should be greater than or equal to chunk size else we set it to chunk size. + [agent_settings.prometheus_fbit_settings] + tcp_listener_chunk_size = 10 + tcp_listener_buffer_size = 10 + tcp_listener_mem_buf_limit = 200 +``` From 0bd136f0a9033206f129b6402a69522953d86aa6 Mon Sep 17 00:00:00 2001 From: Rashmi Chandrashekar Date: Mon, 14 Jun 2021 11:58:26 -0700 Subject: [PATCH 11/12] formatting --- Documentation/AgentSettings/ReadMe.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/AgentSettings/ReadMe.md b/Documentation/AgentSettings/ReadMe.md index cb6dcb7f6..3e55d7d44 100644 --- a/Documentation/AgentSettings/ReadMe.md +++ b/Documentation/AgentSettings/ReadMe.md @@ -5,15 +5,15 @@ The metrics scraped from the telegraf plugin are sent to the fluent bit tcp list In order to support higher volumes of prometheus metrics scraping some of the tcp listener settings can be tuned. [Fluent Bit TCP listener](https://docs.fluentbit.io/manual/pipeline/inputs/tcp) -Chunk Size - This can be increased to process bigger chunks of data. +* Chunk Size - This can be increased to process bigger chunks of data. -Buffer Size - This should be greater than or equal to the chunk size. +* Buffer Size - This should be greater than or equal to the chunk size. -Mem Buf Limit - This can be increased to increase the buffer size. But the memory limit on the sidecar also needs to be increased accordingly. +* Mem Buf Limit - This can be increased to increase the buffer size. But the memory limit on the sidecar also needs to be increased accordingly. Note that this can only be achieved using helm chart today. -* Note - The LA ingestion team also states that higher chunk sizes might not necessarily mean higher throughput since there are pipeline limitations. +** Note - The LA ingestion team also states that higher chunk sizes might not necessarily mean higher throughput since there are pipeline limitations. ``` agent-settings: |- From a06458b1d5361d033cab1b3a02d9065d4cb08c4e Mon Sep 17 00:00:00 2001 From: Rashmi Chandrashekar Date: Mon, 14 Jun 2021 12:00:45 -0700 Subject: [PATCH 12/12] doc link update --- kubernetes/container-azm-ms-agentconfig.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index 3b47e0262..21b31f76f 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -136,6 +136,7 @@ data: collect_basic_metrics = false collect_advanced_metrics = false +# Doc - https://github.com/microsoft/Docker-Provider/blob/ci_prod/Documentation/AgentSettings/ReadMe.md agent-settings: |- # prometheus scrape fluent bit settings for high scale # buffer size should be greater than or equal to chunk size else we set it to chunk size.