Prometheus sidecar memory optimization

.trivyignore

@@ -13,4 +13,7 @@ CVE-2021-43809
 CVE-2021-41816
 CVE-2021-41819
 CVE-2021-31799
-CVE-2021-28965
+CVE-2021-28965
+
+#dpkg vulnerability in ubuntu
+CVE-2022-1664

build/common/installer/scripts/tomlparser-prom-customconfig.rb

@@ -323,7 +323,7 @@ def populateSettingValuesFromConfigMap(parsedConfig)
 
             File.open(file_name, "w") { |file| file.puts new_contents }
             puts "config::Successfully substituted the placeholders in telegraf conf file for custom prometheus scraping"
-            #Set environment variables for telemetry in the sidecar container
+            #Set environment variables for configuration and telemetry in the sidecar container
             if (!@containerType.nil? && @containerType.casecmp(@promSideCar) == 0)
               file = File.open("telemetry_prom_config_env_var", "w")
               if !file.nil?

build/linux/installer/scripts/livenessprobe.sh

@@ -1,4 +1,28 @@
-#!/bin/bash
+#!/bin/bash 
+source /opt/env_vars
+
+if [ -s "inotifyoutput.txt" ]
+then
+  # inotifyoutput file has data(config map was applied)
+  echo "inotifyoutput.txt has been updated - config changed" > /dev/termination-log
+  exit 1
+fi
+
+# Perform the following check only for prometheus sidecar that does OSM scraping or for replicaset when sidecar scraping is disabled
+if [[ ( ( ! -e "/etc/config/kube.conf" ) && ( "${CONTAINER_TYPE}" == "PrometheusSidecar" ) ) ||
+      ( ( -e "/etc/config/kube.conf" ) && ( ( ! -z "${SIDECAR_SCRAPING_ENABLED}" ) && ( "${SIDECAR_SCRAPING_ENABLED}" == "false" ) ) ) ]]; then
+    if [ -s "inotifyoutput-osm.txt" ]
+    then
+      # inotifyoutput-osm file has data(config map was applied)
+      echo "inotifyoutput-osm.txt has been updated - config changed" > /dev/termination-log
+      exit 1
+    fi
+fi
+
+# if this is the prometheus sidecar and there are no prometheus metrics to scrape then the rest of the liveness probe doesn't apply
+if [[ "${CONTAINER_TYPE}" == "PrometheusSidecar" && "${MUTE_PROM_SIDECAR}" == "true" ]]; then
+  exit 0
+fi
 
 #test to exit non zero value if mdsd is not running
 (ps -ef | grep "mdsd" | grep -v "grep")
@@ -53,22 +77,4 @@ then
  # exit 1
 fi
 
-if [ -s "inotifyoutput.txt" ]
-then
-  # inotifyoutput file has data(config map was applied)
-  echo "inotifyoutput.txt has been updated - config changed" > /dev/termination-log
-  exit 1
-fi
-
-# Perform the following check only for prometheus sidecar that does OSM scraping or for replicaset when sidecar scraping is disabled
-if [[ ( ( ! -e "/etc/config/kube.conf" ) && ( "${CONTAINER_TYPE}" == "PrometheusSidecar" ) ) ||
-      ( ( -e "/etc/config/kube.conf" ) && ( ( ! -z "${SIDECAR_SCRAPING_ENABLED}" ) && ( "${SIDECAR_SCRAPING_ENABLED}" == "false" ) ) ) ]]; then
-    if [ -s "inotifyoutput-osm.txt" ]
-    then
-      # inotifyoutput-osm file has data(config map was applied)
-      echo "inotifyoutput-osm.txt has been updated - config changed" > /dev/termination-log
-      exit 1
-    fi
-fi
-
 exit 0

kubernetes/linux/main.sh

@@ -1,5 +1,13 @@
 #!/bin/bash
 
+# please use this instead of adding env vars to bashrc directly
+# usage: setGlobalEnvVar ENABLE_SIDECAR_SCRAPING true
+setGlobalEnvVar() {
+      export "$1"="$2"
+      echo "export \"$1\"=\"$2\"" >> /opt/env_vars
+}
+echo "source /opt/env_vars" >> ~/.bashrc
+
 waitforlisteneronTCPport() {
       local sleepdurationsecs=1
       local totalsleptsecs=0
@@ -366,7 +374,6 @@ if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then
       /usr/bin/ruby2.7 tomlparser-agent-config.rb
 
       cat agent_config_env_var | while read line; do
-            #echo $line
             echo $line >> ~/.bashrc
       done
       source agent_config_env_var
@@ -375,7 +382,6 @@ if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then
       /usr/bin/ruby2.7 tomlparser-npm-config.rb
 
       cat integration_npm_config_env_var | while read line; do
-            #echo $line
             echo $line >> ~/.bashrc
       done
       source integration_npm_config_env_var
@@ -409,7 +415,7 @@ else
       source defaultpromenvvariables-rs
 fi
 
-#Sourcing telemetry environment variable file if it exists
+#Sourcing environment variable file if it exists. This file has telemetry and whether kubernetes pods are monitored
 if [ -e "telemetry_prom_config_env_var" ]; then
       cat telemetry_prom_config_env_var | while read line; do
             echo $line >> ~/.bashrc
@@ -464,6 +470,17 @@ if [[ ( ( ! -e "/etc/config/kube.conf" ) && ( "${CONTAINER_TYPE}" == "Prometheus
       fi
 fi
 
+# If the prometheus sidecar isn't doing anything then there's no need to run mdsd and telegraf in it. 
+if [[ ( "${CONTAINER_TYPE}" == "PrometheusSidecar" ) && 
+      ( "${TELEMETRY_CUSTOM_PROM_MONITOR_PODS}" == "false" ) && 
+      ( "${TELEMETRY_OSM_CONFIGURATION_NAMESPACES_COUNT}" -eq 0 ) ]]; then
+      setGlobalEnvVar MUTE_PROM_SIDECAR true
+else
+      setGlobalEnvVar MUTE_PROM_SIDECAR false
+fi
+
+echo "MUTE_PROM_SIDECAR = $MUTE_PROM_SIDECAR"
+
 #Setting environment variable for CAdvisor metrics to use port 10255/10250 based on curl request
 echo "Making wget request to cadvisor endpoint with port 10250"
 #Defaults to use secure port: 10250
@@ -576,7 +593,7 @@ MDSD_AAD_MSI_AUTH_ARGS=""
 # check if its AAD Auth MSI mode via USING_AAD_MSI_AUTH
 export AAD_MSI_AUTH_MODE=false
 if [ "${USING_AAD_MSI_AUTH}" == "true" ]; then
-   echo "*** activating oneagent in aad auth msi mode ***"
+   echo "*** setting up oneagent in aad auth msi mode ***"
    # msi auth specific args
    MDSD_AAD_MSI_AUTH_ARGS="-a -A"
    export AAD_MSI_AUTH_MODE=true
@@ -593,7 +610,7 @@ if [ "${USING_AAD_MSI_AUTH}" == "true" ]; then
    export MDSD_USE_LOCAL_PERSISTENCY="false"
    echo "export MDSD_USE_LOCAL_PERSISTENCY=$MDSD_USE_LOCAL_PERSISTENCY" >> ~/.bashrc
 else
-  echo "*** activating oneagent in legacy auth mode ***"
+  echo "*** setting up oneagent in legacy auth mode ***"
   CIWORKSPACE_id="$(cat /etc/omsagent-secret/WSID)"
   #use the file path as its secure than env
   CIWORKSPACE_keyFile="/etc/omsagent-secret/KEY"
@@ -617,17 +634,21 @@ source ~/.bashrc
 dpkg -l | grep mdsd | awk '{print $2 " " $3}'
 
 if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then
-    echo "starting mdsd with mdsd-port=26130, fluentport=26230 and influxport=26330 in sidecar container..."
-    #use tenant name to avoid unix socket conflict and different ports for port conflict
-    #roleprefix to use container specific mdsd socket
-    export TENANT_NAME="${CONTAINER_TYPE}"
-    echo "export TENANT_NAME=$TENANT_NAME" >> ~/.bashrc
-    export MDSD_ROLE_PREFIX=/var/run/mdsd-${CONTAINER_TYPE}/default
-    echo "export MDSD_ROLE_PREFIX=$MDSD_ROLE_PREFIX" >> ~/.bashrc
-    source ~/.bashrc
-    mkdir /var/run/mdsd-${CONTAINER_TYPE}
-    # add -T 0xFFFF for full traces
-    mdsd ${MDSD_AAD_MSI_AUTH_ARGS} -r ${MDSD_ROLE_PREFIX} -p 26130 -f 26230 -i 26330 -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos &
+    if [ "${MUTE_PROM_SIDECAR}" != "true" ]; then
+      echo "starting mdsd with mdsd-port=26130, fluentport=26230 and influxport=26330 in sidecar container..."
+      #use tenant name to avoid unix socket conflict and different ports for port conflict
+      #roleprefix to use container specific mdsd socket
+      export TENANT_NAME="${CONTAINER_TYPE}"
+      echo "export TENANT_NAME=$TENANT_NAME" >> ~/.bashrc
+      export MDSD_ROLE_PREFIX=/var/run/mdsd-${CONTAINER_TYPE}/default
+      echo "export MDSD_ROLE_PREFIX=$MDSD_ROLE_PREFIX" >> ~/.bashrc
+      source ~/.bashrc
+      mkdir /var/run/mdsd-${CONTAINER_TYPE}
+      # add -T 0xFFFF for full traces
+      mdsd ${MDSD_AAD_MSI_AUTH_ARGS} -r ${MDSD_ROLE_PREFIX} -p 26130 -f 26230 -i 26330 -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos &
+    else
+      echo "not starting mdsd (no metrics to scrape since MUTE_PROM_SIDECAR is true)"
+    fi
 else
     echo "starting mdsd mode in main container..."
     # add -T 0xFFFF for full traces
@@ -654,13 +675,17 @@ fi
 #If config parsing was successful, a copy of the conf file with replaced custom settings file is created
 if [ ! -e "/etc/config/kube.conf" ]; then
       if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ] && [ -e "/opt/telegraf-test-prom-side-car.conf" ]; then
-            echo "****************Start Telegraf in Test Mode**************************"
-            /opt/telegraf --config /opt/telegraf-test-prom-side-car.conf --input-filter file -test
-            if [ $? -eq 0 ]; then
-                  mv "/opt/telegraf-test-prom-side-car.conf" "/etc/opt/microsoft/docker-cimprov/telegraf-prom-side-car.conf"
-                  echo "Moving test conf file to telegraf side-car conf since test run succeeded"
+            if [ "${MUTE_PROM_SIDECAR}" != "true" ]; then
+                  echo "****************Start Telegraf in Test Mode**************************"
+                  /opt/telegraf --config /opt/telegraf-test-prom-side-car.conf --input-filter file -test
+                  if [ $? -eq 0 ]; then
+                        mv "/opt/telegraf-test-prom-side-car.conf" "/etc/opt/microsoft/docker-cimprov/telegraf-prom-side-car.conf"
+                        echo "Moving test conf file to telegraf side-car conf since test run succeeded"
+                  fi
+                  echo "****************End Telegraf Run in Test Mode**************************"
+            else
+                  echo "****************Skipping Telegraf Run in Test Mode since MUTE_PROM_SIDECAR is true**************************"
             fi
-            echo "****************End Telegraf Run in Test Mode**************************"
       else
             if [ -e "/opt/telegraf-test.conf" ]; then
                   echo "****************Start Telegraf in Test Mode**************************"
@@ -687,9 +712,13 @@ fi
 #telegraf & fluentbit requirements
 if [ ! -e "/etc/config/kube.conf" ]; then
       if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then
-            echo "starting fluent-bit and setting telegraf conf file for prometheus sidecar"
-            /opt/td-agent-bit/bin/td-agent-bit -c /etc/opt/microsoft/docker-cimprov/td-agent-bit-prom-side-car.conf -e /opt/td-agent-bit/bin/out_oms.so &
             telegrafConfFile="/etc/opt/microsoft/docker-cimprov/telegraf-prom-side-car.conf"
+            if [ "${MUTE_PROM_SIDECAR}" != "true" ]; then
+                  echo "starting fluent-bit and setting telegraf conf file for prometheus sidecar"
+                  /opt/td-agent-bit/bin/td-agent-bit -c /etc/opt/microsoft/docker-cimprov/td-agent-bit-prom-side-car.conf -e /opt/td-agent-bit/bin/out_oms.so &
+            else
+                  echo "not starting fluent-bit in prometheus sidecar (no metrics to scrape since MUTE_PROM_SIDECAR is true)"
+            fi
       else
             echo "starting fluent-bit and setting telegraf conf file for daemonset"
             if [ "$CONTAINER_RUNTIME" == "docker" ]; then
@@ -756,8 +785,12 @@ echo "export HOST_VAR=/hostfs/var" >> ~/.bashrc
 
 if [ ! -e "/etc/config/kube.conf" ]; then
       if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then
-            echo "checking for listener on tcp #25229 and waiting for 30 secs if not.."
-            waitforlisteneronTCPport 25229 30
+            if [ "${MUTE_PROM_SIDECAR}" != "true" ]; then
+                  echo "checking for listener on tcp #25229 and waiting for 30 secs if not.."
+                  waitforlisteneronTCPport 25229 30
+            else
+                  echo "no metrics to scrape since MUTE_PROM_SIDECAR is true, not checking for listener on tcp #25229"
+            fi
       else
             echo "checking for listener on tcp #25226 and waiting for 30 secs if not.."
             waitforlisteneronTCPport 25226 30
@@ -769,10 +802,15 @@ else
       waitforlisteneronTCPport 25226 30
 fi
 
+
 #start telegraf
-/opt/telegraf --config $telegrafConfFile &
-/opt/telegraf --version
-dpkg -l | grep td-agent-bit | awk '{print $2 " " $3}'
+if [ "${MUTE_PROM_SIDECAR}" != "true" ]; then
+      /opt/telegraf --config $telegrafConfFile &
+      echo "telegraf version: $(/opt/telegraf --version)"
+      dpkg -l | grep td-agent-bit | awk '{print $2 " " $3}'
+else
+      echo "not starting telegraf (no metrics to scrape since MUTE_PROM_SIDECAR is true)"
+fi
 
 #dpkg -l | grep telegraf | awk '{print $2 " " $3}'
 
@@ -785,7 +823,11 @@ service rsyslog stop
 echo "getting rsyslog status..."
 service rsyslog status
 
-checkAgentOnboardingStatus $AAD_MSI_AUTH_MODE 30
+if [ "${MUTE_PROM_SIDECAR}" != "true" ]; then
+      checkAgentOnboardingStatus $AAD_MSI_AUTH_MODE 30
+else
+      echo "not checking onboarding status (no metrics to scrape since MUTE_PROM_SIDECAR is true)"
+fi
 
 shutdown() {
 	 pkill -f mdsd