From 86c6ea7e92c4729fe0bfe679aef6a8d1f680523c Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Thu, 28 Apr 2022 17:01:10 +0000 Subject: [PATCH 01/36] update telegraf to 1.22.2 to fix vulns --- .trivyignore | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.trivyignore b/.trivyignore index 3a8089422..bb0f415dc 100644 --- a/.trivyignore +++ b/.trivyignore @@ -8,9 +8,9 @@ CVE-2022-27191 CVE-2021-42836 # ruby in /usr/lib -CVE-2020-36327 -CVE-2021-43809 -CVE-2021-41816 -CVE-2021-41819 -CVE-2021-31799 -CVE-2021-28965 \ No newline at end of file +# CVE-2020-36327 +# CVE-2021-43809 +# CVE-2021-41816 +# CVE-2021-41819 +# CVE-2021-31799 +# CVE-2021-28965 \ No newline at end of file From 55669040799c7d148a0cb87304c82af4c803cab8 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Thu, 28 Apr 2022 17:22:48 +0000 Subject: [PATCH 02/36] remove changes committed by mistake --- .trivyignore | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.trivyignore b/.trivyignore index bb0f415dc..3a8089422 100644 --- a/.trivyignore +++ b/.trivyignore @@ -8,9 +8,9 @@ CVE-2022-27191 CVE-2021-42836 # ruby in /usr/lib -# CVE-2020-36327 -# CVE-2021-43809 -# CVE-2021-41816 -# CVE-2021-41819 -# CVE-2021-31799 -# CVE-2021-28965 \ No newline at end of file +CVE-2020-36327 +CVE-2021-43809 +CVE-2021-41816 +CVE-2021-41819 +CVE-2021-31799 +CVE-2021-28965 \ No newline at end of file From 30ad904987104d681c49bfb4f13dcced140bc107 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Mon, 2 May 2022 22:50:24 +0000 Subject: [PATCH 03/36] test --- .trivyignore | 6 ------ kubernetes/linux/Dockerfile.multiarch | 3 ++- kubernetes/linux/setup.sh | 16 ++++++++++++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.trivyignore b/.trivyignore index 3a8089422..b72133435 100644 --- a/.trivyignore +++ b/.trivyignore @@ -8,9 +8,3 @@ CVE-2022-27191 CVE-2021-42836 # ruby in /usr/lib -CVE-2020-36327 -CVE-2021-43809 -CVE-2021-41816 -CVE-2021-41819 -CVE-2021-31799 -CVE-2021-28965 \ No newline at end of file diff --git a/kubernetes/linux/Dockerfile.multiarch b/kubernetes/linux/Dockerfile.multiarch index 38103dd65..4b6498314 100644 --- a/kubernetes/linux/Dockerfile.multiarch +++ b/kubernetes/linux/Dockerfile.multiarch @@ -24,7 +24,7 @@ ENV AZMON_COLLECT_ENV False ENV KUBE_CLIENT_BACKOFF_BASE 1 ENV KUBE_CLIENT_BACKOFF_DURATION 0 ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9 -RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg make && rm -rf /var/lib/apt/lists/* +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg2 make && rm -rf /var/lib/apt/lists/* COPY --from=builder /src/kubernetes/linux/Linux_ULINUX_1.0_*_64_Release/docker-cimprov-*.*.*-*.*.sh $tmpdir/ COPY kubernetes/linux/setup.sh kubernetes/linux/main.sh kubernetes/linux/defaultpromenvvariables kubernetes/linux/defaultpromenvvariables-rs kubernetes/linux/defaultpromenvvariables-sidecar kubernetes/linux/mdsd.xml kubernetes/linux/envmdsd kubernetes/linux/logrotate.conf $tmpdir/ @@ -34,6 +34,7 @@ ENV AGENT_VERSION ${IMAGE_TAG} WORKDIR ${tmpdir} +SHELL [ "/bin/bash", "-l", "-c" ] RUN chmod 775 $tmpdir/*.sh; sync; $tmpdir/setup.sh ${TARGETARCH} # Do vulnerability scan in a seperate stage to avoid adding layer diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index c478af0e5..c54528b23 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -11,6 +11,12 @@ fi #upgrade apt to latest version apt-get update && apt-get install -y apt && DEBIAN_FRONTEND=noninteractive apt-get install -y locales +gpg2 --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB +curl -sSL https://get.rvm.io | bash -s +source /etc/profile.d/rvm.sh +rvm install 3.1.1 +rvm --default use 3.1.1 + sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ update-locale LANG=en_US.UTF-8 @@ -57,10 +63,12 @@ sudo apt-get update sudo apt-get install td-agent-bit=1.7.8 -y # install ruby2.7 -sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F5DA5F09C3173AA6 -sudo echo "deb http://ppa.launchpad.net/brightbox/ruby-ng/ubuntu bionic main" >> /etc/apt/sources.list -sudo apt-get update -sudo apt-get install ruby2.7 ruby2.7-dev gcc make -y +# sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F5DA5F09C3173AA6 +# sudo echo "deb http://ppa.launchpad.net/brightbox/ruby-ng/ubuntu bionic main" >> /etc/apt/sources.list +# sudo apt-get update +# sudo apt-get install ruby2.7 ruby2.7-dev gcc make -y + + # fluentd v1 gem gem install fluentd -v "1.14.2" --no-document fluentd --setup ./fluent From 6e88588c0b2c53e291b82055554f1390cf4aedb4 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 3 May 2022 00:05:35 +0000 Subject: [PATCH 04/36] use precompiled version --- kubernetes/linux/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index c54528b23..fd708d212 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -14,8 +14,8 @@ apt-get update && apt-get install -y apt && DEBIAN_FRONTEND=noninteractive apt-g gpg2 --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB curl -sSL https://get.rvm.io | bash -s source /etc/profile.d/rvm.sh -rvm install 3.1.1 -rvm --default use 3.1.1 +rvm install 3.1.0 +rvm --default use 3.1.0 sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ From 34f5b6180a6819046782dab02a8032d0d084232a Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 3 May 2022 01:09:17 +0000 Subject: [PATCH 05/36] change paths --- kubernetes/linux/Dockerfile.multiarch | 1 - kubernetes/linux/main.sh | 21 +++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/kubernetes/linux/Dockerfile.multiarch b/kubernetes/linux/Dockerfile.multiarch index 4b6498314..f5b4cdd60 100644 --- a/kubernetes/linux/Dockerfile.multiarch +++ b/kubernetes/linux/Dockerfile.multiarch @@ -34,7 +34,6 @@ ENV AGENT_VERSION ${IMAGE_TAG} WORKDIR ${tmpdir} -SHELL [ "/bin/bash", "-l", "-c" ] RUN chmod 775 $tmpdir/*.sh; sync; $tmpdir/setup.sh ${TARGETARCH} # Do vulnerability scan in a seperate stage to avoid adding layer diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index 2b25b044c..10823736b 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -82,6 +82,7 @@ checkAgentOnboardingStatus() { fi } +source /etc/profile.d/rvm.sh #using /var/opt/microsoft/docker-cimprov/state instead of /var/opt/microsoft/omsagent/state since the latter gets deleted during onboarding mkdir -p /var/opt/microsoft/docker-cimprov/state @@ -352,7 +353,7 @@ source ~/.bashrc if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then #Parse the configmap to set the right environment variables. - /usr/bin/ruby2.7 tomlparser.rb + ruby tomlparser.rb cat config_env_var | while read line; do echo $line >> ~/.bashrc @@ -363,7 +364,7 @@ fi #Parse the configmap to set the right environment variables for agent config. #Note > tomlparser-agent-config.rb has to be parsed first before td-agent-bit-conf-customizer.rb for fbit agent settings if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then - /usr/bin/ruby2.7 tomlparser-agent-config.rb + ruby tomlparser-agent-config.rb cat agent_config_env_var | while read line; do #echo $line @@ -372,7 +373,7 @@ if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then source agent_config_env_var #Parse the configmap to set the right environment variables for network policy manager (npm) integration. - /usr/bin/ruby2.7 tomlparser-npm-config.rb + ruby tomlparser-npm-config.rb cat integration_npm_config_env_var | while read line; do #echo $line @@ -383,11 +384,11 @@ fi #Replace the placeholders in td-agent-bit.conf file for fluentbit with custom/default values in daemonset if [ ! -e "/etc/config/kube.conf" ] && [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then - /usr/bin/ruby2.7 td-agent-bit-conf-customizer.rb + ruby td-agent-bit-conf-customizer.rb fi #Parse the prometheus configmap to create a file with new custom settings. -/usr/bin/ruby2.7 tomlparser-prom-customconfig.rb +ruby tomlparser-prom-customconfig.rb #Setting default environment variables to be used in any case of failure in the above steps if [ ! -e "/etc/config/kube.conf" ]; then @@ -421,7 +422,7 @@ fi if [ ! -e "/etc/config/kube.conf" ]; then if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then #Parse the agent configmap to create a file with new custom settings. - /usr/bin/ruby2.7 tomlparser-prom-agent-config.rb + ruby tomlparser-prom-agent-config.rb #Sourcing config environment variable file if it exists if [ -e "side_car_fbit_config_env_var" ]; then cat side_car_fbit_config_env_var | while read line; do @@ -435,7 +436,7 @@ fi #Parse the configmap to set the right environment variables for MDM metrics configuration for Alerting. if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then - /usr/bin/ruby2.7 tomlparser-mdm-metrics-config.rb + ruby tomlparser-mdm-metrics-config.rb cat config_mdm_metrics_env_var | while read line; do echo $line >> ~/.bashrc @@ -443,7 +444,7 @@ if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then source config_mdm_metrics_env_var #Parse the configmap to set the right environment variables for metric collection settings - /usr/bin/ruby2.7 tomlparser-metric-collection-config.rb + ruby tomlparser-metric-collection-config.rb cat config_metric_collection_env_var | while read line; do echo $line >> ~/.bashrc @@ -454,7 +455,7 @@ fi # OSM scraping to be done in replicaset if sidecar car scraping is disabled and always do the scraping from the sidecar (It will always be either one of the two) if [[ ( ( ! -e "/etc/config/kube.conf" ) && ( "${CONTAINER_TYPE}" == "PrometheusSidecar" ) ) || ( ( -e "/etc/config/kube.conf" ) && ( "${SIDECAR_SCRAPING_ENABLED}" == "false" ) ) ]]; then - /usr/bin/ruby2.7 tomlparser-osm-config.rb + ruby tomlparser-osm-config.rb if [ -e "integration_osm_config_env_var" ]; then cat integration_osm_config_env_var | while read line; do @@ -541,7 +542,7 @@ if [ "$CONTAINER_RUNTIME" != "docker" ]; then fi echo "set caps for ruby process to read container env from proc" -sudo setcap cap_sys_ptrace,cap_dac_read_search+ep /usr/bin/ruby2.7 +sudo setcap cap_sys_ptrace,cap_dac_read_search+ep ruby echo "export KUBELET_RUNTIME_OPERATIONS_METRIC="$KUBELET_RUNTIME_OPERATIONS_METRIC >> ~/.bashrc echo "export KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC="$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC >> ~/.bashrc From ce49fc3367b76f2ad3f2bb127506ecbb799fd9f3 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 3 May 2022 01:58:34 +0000 Subject: [PATCH 06/36] run as bash shell --- kubernetes/linux/setup.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index fd708d212..982c514f4 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -1,3 +1,5 @@ +#!/bin/bash + TMPDIR="/opt" cd $TMPDIR From 2375bbe918d6a814180b7467b00a023c3a6c3145 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 3 May 2022 18:25:20 +0000 Subject: [PATCH 07/36] update ruby in windows --- kubernetes/windows/Dockerfile | 6 +++--- kubernetes/windows/Dockerfile-dev-base-image | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kubernetes/windows/Dockerfile b/kubernetes/windows/Dockerfile index 672dfb1f7..5f766fd57 100644 --- a/kubernetes/windows/Dockerfile +++ b/kubernetes/windows/Dockerfile @@ -11,8 +11,8 @@ ARG IMAGE_TAG=win-ciprod03172022 # Docker creates a layer for every RUN-Statement RUN powershell -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" # Fluentd depends on cool.io whose fat gem is only available for Ruby < 2.5, so need to specify --platform ruby when install Ruby > 2.5 and install msys2 to get dev tools -RUN choco install -y ruby --version 2.7.5.1 --params "'/InstallDir:C:\ruby27'" \ -&& choco install -y msys2 --version 20211130.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby27\msys64'" \ +RUN choco install -y ruby --version 3.1.0.1 --params "'/InstallDir:C:\ruby31'" \ +&& choco install -y msys2 --version 20211130.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby31\msys64'" \ && choco install -y vim # gangams - optional MSYS2 update via ridk failing in merged docker file so skipping that since we dont need optional update @@ -32,7 +32,7 @@ RUN refreshenv \ && gem sources --clear-all # Remove gem cache and chocolatey -RUN powershell -Command "Remove-Item -Force C:\ruby27\lib\ruby\gems\2.7.0\cache\*.gem; Remove-Item -Recurse -Force 'C:\ProgramData\chocolatey'" +RUN powershell -Command "Remove-Item -Force C:\ruby31\lib\ruby\gems\3.1.0\cache\*.gem; Remove-Item -Recurse -Force 'C:\ProgramData\chocolatey'" SHELL ["powershell"] diff --git a/kubernetes/windows/Dockerfile-dev-base-image b/kubernetes/windows/Dockerfile-dev-base-image index 3aca6ae20..89452256d 100644 --- a/kubernetes/windows/Dockerfile-dev-base-image +++ b/kubernetes/windows/Dockerfile-dev-base-image @@ -8,8 +8,8 @@ LABEL vendor=Microsoft\ Corp \ # Docker creates a layer for every RUN-Statement RUN powershell -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" # Fluentd depends on cool.io whose fat gem is only available for Ruby < 2.5, so need to specify --platform ruby when install Ruby > 2.5 and install msys2 to get dev tools -RUN choco install -y ruby --version 2.7.5.1 --params "'/InstallDir:C:\ruby27'" \ -&& choco install -y msys2 --version 20210604.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby27\msys64'" \ +RUN choco install -y ruby --version 3.1.0.1 --params "'/InstallDir:C:\ruby31'" \ +&& choco install -y msys2 --version 20210604.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby31\msys64'" \ && choco install -y vim # gangams - optional MSYS2 update via ridk failing in merged docker file so skipping that since we dont need optional update @@ -29,7 +29,7 @@ RUN refreshenv \ && gem sources --clear-all # Remove gem cache and chocolatey -RUN powershell -Command "Remove-Item -Force C:\ruby27\lib\ruby\gems\2.7.0\cache\*.gem; Remove-Item -Recurse -Force 'C:\ProgramData\chocolatey'" +RUN powershell -Command "Remove-Item -Force C:\ruby31\lib\ruby\gems\3.1.0\cache\*.gem; Remove-Item -Recurse -Force 'C:\ProgramData\chocolatey'" SHELL ["powershell"] From fa92dabed9ec0e4313971491cde492adc8fbb5f2 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 3 May 2022 13:18:28 -0700 Subject: [PATCH 08/36] update cool.io to build with ruby 3.1.0 --- kubernetes/windows/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/windows/Dockerfile b/kubernetes/windows/Dockerfile index 5f766fd57..b22c388cb 100644 --- a/kubernetes/windows/Dockerfile +++ b/kubernetes/windows/Dockerfile @@ -19,7 +19,7 @@ RUN choco install -y ruby --version 3.1.0.1 --params "'/InstallDir:C:\ruby31'" \ RUN refreshenv \ && ridk install 3 \ && echo gem: --no-document >> C:\ProgramData\gemrc \ -&& gem install cool.io -v 1.5.4 --platform ruby \ +&& gem install cool.io -v 1.7.1 --platform ruby \ && gem install oj -v 3.3.10 \ && gem install json -v 2.2.0 \ && gem install fluentd -v 1.14.2 \ From 3226f27ee5866d342c4e27e1ef722125b4c9495f Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 3 May 2022 21:36:19 +0000 Subject: [PATCH 09/36] ruby paths --- kubernetes/linux/main.sh | 3 ++- kubernetes/linux/setup.sh | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index 10823736b..d789fcdce 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -82,7 +82,8 @@ checkAgentOnboardingStatus() { fi } -source /etc/profile.d/rvm.sh +# setup paths for ruby +source ~/.bashrc #using /var/opt/microsoft/docker-cimprov/state instead of /var/opt/microsoft/omsagent/state since the latter gets deleted during onboarding mkdir -p /var/opt/microsoft/docker-cimprov/state diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 982c514f4..3926acbef 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -13,9 +13,17 @@ fi #upgrade apt to latest version apt-get update && apt-get install -y apt && DEBIAN_FRONTEND=noninteractive apt-get install -y locales -gpg2 --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB -curl -sSL https://get.rvm.io | bash -s -source /etc/profile.d/rvm.sh + +curl -sSL https://rvm.io/mpapis.asc | gpg --import - +curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - +curl -sSL https://get.rvm.io | bash -s stable + +# setup paths for ruby and rvm +if [ -f /etc/profile.d/rvm.sh ]; then + source /etc/profile.d/rvm.sh + echo "[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh" >> ~/.bashrc +fi + rvm install 3.1.0 rvm --default use 3.1.0 @@ -85,6 +93,7 @@ rm -f $TMPDIR/telegraf-*.tar.gz # remove build dependencies sudo apt-get remove ruby2.7-dev gcc make -y +sudo apt autoremove -y # Remove settings for cron.daily that conflict with the node's cron.daily. Since both are trying to rotate the same files # in /var/log at the same time, the rotation doesn't happen correctly and then the *.1 file is forever logged to. From 0de200972529246add52fe831942390566af2bca Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 3 May 2022 22:50:16 +0000 Subject: [PATCH 10/36] fix path in main --- kubernetes/linux/main.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index d789fcdce..68198959c 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -83,7 +83,7 @@ checkAgentOnboardingStatus() { } # setup paths for ruby -source ~/.bashrc +[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh #using /var/opt/microsoft/docker-cimprov/state instead of /var/opt/microsoft/omsagent/state since the latter gets deleted during onboarding mkdir -p /var/opt/microsoft/docker-cimprov/state From 73dcd1c62ca202b73991886007497f1305987ded Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 3 May 2022 23:54:17 +0000 Subject: [PATCH 11/36] test --- kubernetes/linux/main.sh | 3 ++- kubernetes/windows/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index 68198959c..f1cbbd1db 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -543,7 +543,8 @@ if [ "$CONTAINER_RUNTIME" != "docker" ]; then fi echo "set caps for ruby process to read container env from proc" -sudo setcap cap_sys_ptrace,cap_dac_read_search+ep ruby +RUBY_PATH=$(which ruby) +sudo setcap cap_sys_ptrace,cap_dac_read_search+ep "$RUBY_PATH" echo "export KUBELET_RUNTIME_OPERATIONS_METRIC="$KUBELET_RUNTIME_OPERATIONS_METRIC >> ~/.bashrc echo "export KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC="$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC >> ~/.bashrc diff --git a/kubernetes/windows/Dockerfile b/kubernetes/windows/Dockerfile index b22c388cb..3c125e178 100644 --- a/kubernetes/windows/Dockerfile +++ b/kubernetes/windows/Dockerfile @@ -22,7 +22,7 @@ RUN refreshenv \ && gem install cool.io -v 1.7.1 --platform ruby \ && gem install oj -v 3.3.10 \ && gem install json -v 2.2.0 \ -&& gem install fluentd -v 1.14.2 \ +&& gem install fluentd -v 1.14.6 \ && gem install win32-service -v 1.0.1 \ && gem install win32-ipc -v 0.7.0 \ && gem install win32-event -v 0.6.3 \ From 06821d6b40c1995d4fb1c324ae845c55364c1ad8 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Wed, 4 May 2022 05:23:22 +0000 Subject: [PATCH 12/36] bump fluentd for linux --- .trivyignore | 4 +--- kubernetes/linux/setup.sh | 11 ++--------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/.trivyignore b/.trivyignore index b72133435..d8be7f39e 100644 --- a/.trivyignore +++ b/.trivyignore @@ -5,6 +5,4 @@ CVE-2019-3826 CVE-2022-27191 #still present in mdsd telegraf -CVE-2021-42836 - -# ruby in /usr/lib +CVE-2021-42836 \ No newline at end of file diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 3926acbef..068854976 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -72,15 +72,8 @@ sudo echo "deb https://packages.fluentbit.io/ubuntu/bionic bionic main" >> /etc/ sudo apt-get update sudo apt-get install td-agent-bit=1.7.8 -y -# install ruby2.7 -# sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F5DA5F09C3173AA6 -# sudo echo "deb http://ppa.launchpad.net/brightbox/ruby-ng/ubuntu bionic main" >> /etc/apt/sources.list -# sudo apt-get update -# sudo apt-get install ruby2.7 ruby2.7-dev gcc make -y - - # fluentd v1 gem -gem install fluentd -v "1.14.2" --no-document +gem install fluentd -v "1.14.6" --no-document fluentd --setup ./fluent gem install gyoku iso8601 --no-doc @@ -92,7 +85,7 @@ rm -f $TMPDIR/envmdsd rm -f $TMPDIR/telegraf-*.tar.gz # remove build dependencies -sudo apt-get remove ruby2.7-dev gcc make -y +sudo apt-get remove gcc make -y sudo apt autoremove -y # Remove settings for cron.daily that conflict with the node's cron.daily. Since both are trying to rotate the same files From 2d83478aa772b6d9f6e9aa737ff0bd5b68a45ef1 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Wed, 4 May 2022 23:02:02 +0000 Subject: [PATCH 13/36] use tomlrb gem for linux; bump version for windows --- README.md | 1 - .../scripts/tomlparser-agent-config.rb | 6 +- .../scripts/tomlparser-mdm-metrics-config.rb | 6 +- .../scripts/tomlparser-prom-agent-config.rb | 6 +- .../scripts/tomlparser-prom-customconfig.rb | 8 +- build/common/installer/scripts/tomlparser.rb | 6 +- .../installer/datafiles/base_container.data | 9 - .../tomlparser-metric-collection-config.rb | 2 +- .../scripts/tomlparser-npm-config.rb | 6 +- .../scripts/tomlparser-osm-config.rb | 2 +- kubernetes/linux/setup.sh | 1 + kubernetes/windows/Dockerfile | 2 +- source/toml-parser/tomlrb.rb | 44 -- source/toml-parser/tomlrb/generated_parser.rb | 542 ------------------ source/toml-parser/tomlrb/handler.rb | 73 --- source/toml-parser/tomlrb/parser.rb | 18 - source/toml-parser/tomlrb/parser.y | 104 ---- source/toml-parser/tomlrb/scanner.rb | 54 -- source/toml-parser/tomlrb/string_utils.rb | 33 -- source/toml-parser/tomlrb/version.rb | 3 - 20 files changed, 11 insertions(+), 915 deletions(-) delete mode 100644 source/toml-parser/tomlrb.rb delete mode 100644 source/toml-parser/tomlrb/generated_parser.rb delete mode 100644 source/toml-parser/tomlrb/handler.rb delete mode 100644 source/toml-parser/tomlrb/parser.rb delete mode 100644 source/toml-parser/tomlrb/parser.y delete mode 100644 source/toml-parser/tomlrb/scanner.rb delete mode 100644 source/toml-parser/tomlrb/string_utils.rb delete mode 100644 source/toml-parser/tomlrb/version.rb diff --git a/README.md b/README.md index 6e51d256b..571547afc 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,6 @@ The general directory structure is: │ │ | ├── health/ - code for health feature │ │ | ├── lib/ - lib for app insights ruby and this code of application_insights gem │ │ | ... - plugins in, out and filters code in ruby -│ ├── toml-parser/ - code for parsing of toml configuration files ├── test/ - source code for tests │ ├── e2e/ - e2e tests to validate agent and e2e workflow(s) │ ├── unit-tests/ - unit tests code diff --git a/build/common/installer/scripts/tomlparser-agent-config.rb b/build/common/installer/scripts/tomlparser-agent-config.rb index ebe1e3982..a6b6ee0a1 100644 --- a/build/common/installer/scripts/tomlparser-agent-config.rb +++ b/build/common/installer/scripts/tomlparser-agent-config.rb @@ -2,11 +2,7 @@ #this should be require relative in Linux and require in windows, since it is a gem install on windows @os_type = ENV["OS_TYPE"] -if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 - require "tomlrb" -else - require_relative "tomlrb" -end +require "tomlrb" require_relative "ConfigParseErrorLogger" diff --git a/build/common/installer/scripts/tomlparser-mdm-metrics-config.rb b/build/common/installer/scripts/tomlparser-mdm-metrics-config.rb index b6a4419cf..17c1ca118 100644 --- a/build/common/installer/scripts/tomlparser-mdm-metrics-config.rb +++ b/build/common/installer/scripts/tomlparser-mdm-metrics-config.rb @@ -3,11 +3,7 @@ #this should be require relative in Linux and require in windows, since it is a gem install on windows @os_type = ENV["OS_TYPE"] -if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 - require "tomlrb" -else - require_relative "tomlrb" -end +require "tomlrb" require_relative "/etc/fluent/plugin/constants" require_relative "ConfigParseErrorLogger" diff --git a/build/common/installer/scripts/tomlparser-prom-agent-config.rb b/build/common/installer/scripts/tomlparser-prom-agent-config.rb index 664691a44..abc939f52 100644 --- a/build/common/installer/scripts/tomlparser-prom-agent-config.rb +++ b/build/common/installer/scripts/tomlparser-prom-agent-config.rb @@ -2,11 +2,7 @@ #this should be require relative in Linux and require in windows, since it is a gem install on windows @os_type = ENV["OS_TYPE"] -if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 - require "tomlrb" -else - require_relative "tomlrb" -end +require "tomlrb" require_relative "ConfigParseErrorLogger" diff --git a/build/common/installer/scripts/tomlparser-prom-customconfig.rb b/build/common/installer/scripts/tomlparser-prom-customconfig.rb index 819c1956f..375d5778b 100644 --- a/build/common/installer/scripts/tomlparser-prom-customconfig.rb +++ b/build/common/installer/scripts/tomlparser-prom-customconfig.rb @@ -2,12 +2,8 @@ #this should be require relative in Linux and require in windows, since it is a gem install on windows @os_type = ENV["OS_TYPE"] -if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 - require "tomlrb" -else - require_relative "tomlrb" -end -# require_relative "tomlrb" +require "tomlrb" + require_relative "ConfigParseErrorLogger" require "fileutils" diff --git a/build/common/installer/scripts/tomlparser.rb b/build/common/installer/scripts/tomlparser.rb index 6a2f3c6d6..6d3ee6e78 100644 --- a/build/common/installer/scripts/tomlparser.rb +++ b/build/common/installer/scripts/tomlparser.rb @@ -2,11 +2,7 @@ #this should be require relative in Linux and require in windows, since it is a gem install on windows @os_type = ENV["OS_TYPE"] -if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 - require "tomlrb" -else - require_relative "tomlrb" -end +require "tomlrb" require_relative "ConfigParseErrorLogger" diff --git a/build/linux/installer/datafiles/base_container.data b/build/linux/installer/datafiles/base_container.data index 7dcbde31f..98bb3fc1b 100644 --- a/build/linux/installer/datafiles/base_container.data +++ b/build/linux/installer/datafiles/base_container.data @@ -20,15 +20,6 @@ MAINTAINER: 'Microsoft Corporation' /etc/opt/microsoft/docker-cimprov/container.conf; build/linux/installer/conf/container.conf; 644; root; root -/opt/tomlrb.rb; source/toml-parser/tomlrb.rb; 644; root; root -/opt/tomlrb/generated_parser.rb; source/toml-parser/tomlrb/generated_parser.rb; 644; root; root -/opt/tomlrb/handler.rb; source/toml-parser/tomlrb/handler.rb; 644; root; root -/opt/tomlrb/parser.rb; source/toml-parser/tomlrb/parser.rb; 644; root; root -/opt/tomlrb/parser.y; source/toml-parser/tomlrb/parser.y; 644; root; root -/opt/tomlrb/scanner.rb; source/toml-parser/tomlrb/scanner.rb; 644; root; root -/opt/tomlrb/string_utils.rb; source/toml-parser/tomlrb/string_utils.rb; 644; root; root -/opt/tomlrb/version.rb; source/toml-parser/tomlrb/version.rb; 644; root; root - /opt/td-agent-bit/bin/out_oms.so; intermediate/${{BUILD_CONFIGURATION}}/out_oms.so; 755; root; root /etc/opt/microsoft/docker-cimprov/prometheus-side-car.conf; build/linux/installer/conf/prometheus-side-car.conf; 644; root; root /etc/opt/microsoft/docker-cimprov/td-agent-bit.conf; build/linux/installer/conf/td-agent-bit.conf; 644; root; root diff --git a/build/linux/installer/scripts/tomlparser-metric-collection-config.rb b/build/linux/installer/scripts/tomlparser-metric-collection-config.rb index cee41312b..3001fdbaf 100644 --- a/build/linux/installer/scripts/tomlparser-metric-collection-config.rb +++ b/build/linux/installer/scripts/tomlparser-metric-collection-config.rb @@ -1,7 +1,7 @@ #!/usr/local/bin/ruby # frozen_string_literal: true -require_relative "tomlrb" +require "tomlrb" require_relative "ConfigParseErrorLogger" require_relative "/etc/fluent/plugin/constants" diff --git a/build/linux/installer/scripts/tomlparser-npm-config.rb b/build/linux/installer/scripts/tomlparser-npm-config.rb index 777fef209..00fa9fe53 100644 --- a/build/linux/installer/scripts/tomlparser-npm-config.rb +++ b/build/linux/installer/scripts/tomlparser-npm-config.rb @@ -2,11 +2,7 @@ #this should be require relative in Linux and require in windows, since it is a gem install on windows @os_type = ENV["OS_TYPE"] -if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 - require "tomlrb" -else - require_relative "tomlrb" -end +require "tomlrb" require_relative "ConfigParseErrorLogger" diff --git a/build/linux/installer/scripts/tomlparser-osm-config.rb b/build/linux/installer/scripts/tomlparser-osm-config.rb index 096064db8..2ac5ef387 100644 --- a/build/linux/installer/scripts/tomlparser-osm-config.rb +++ b/build/linux/installer/scripts/tomlparser-osm-config.rb @@ -1,6 +1,6 @@ #!/usr/local/bin/ruby -require_relative "tomlrb" +require "tomlrb" require "fileutils" require_relative "ConfigParseErrorLogger" diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 068854976..f43fc602a 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -76,6 +76,7 @@ sudo apt-get install td-agent-bit=1.7.8 -y gem install fluentd -v "1.14.6" --no-document fluentd --setup ./fluent gem install gyoku iso8601 --no-doc +gem install tomlrb -v "2.0.1" --no-document rm -f $TMPDIR/docker-cimprov*.sh diff --git a/kubernetes/windows/Dockerfile b/kubernetes/windows/Dockerfile index 3c125e178..4923a009b 100644 --- a/kubernetes/windows/Dockerfile +++ b/kubernetes/windows/Dockerfile @@ -27,7 +27,7 @@ RUN refreshenv \ && gem install win32-ipc -v 0.7.0 \ && gem install win32-event -v 0.6.3 \ && gem install windows-pr -v 1.2.6 \ -&& gem install tomlrb -v 1.3.0 \ +&& gem install tomlrb -v 2.0.1 \ && gem install gyoku -v 1.3.1 \ && gem sources --clear-all diff --git a/source/toml-parser/tomlrb.rb b/source/toml-parser/tomlrb.rb deleted file mode 100644 index c0eff9093..000000000 --- a/source/toml-parser/tomlrb.rb +++ /dev/null @@ -1,44 +0,0 @@ -require "time" -require "stringio" -require_relative "tomlrb/version" -require_relative "tomlrb/string_utils" -require_relative "tomlrb/scanner" -require_relative "tomlrb/parser" -require_relative "tomlrb/handler" - -module Tomlrb - class ParseError < StandardError; end - - # Parses a valid TOML string into its Ruby data structure - # - # @param string_or_io [String, StringIO] the content - # @param options [Hash] the options hash - # @option options [Boolean] :symbolize_keys (false) whether to return the keys as symbols or strings - # @return [Hash] the Ruby data structure represented by the input - def self.parse(string_or_io, **options) - io = string_or_io.is_a?(String) ? StringIO.new(string_or_io) : string_or_io - scanner = Scanner.new(io) - parser = Parser.new(scanner, options) - begin - handler = parser.parse - rescue Racc::ParseError => e - raise ParseError, e.message - end - - handler.output - end - - # Reads a file content and parses it into its Ruby data structure - # - # @param path [String] the path to the file - # @param options [Hash] the options hash - # @option options [Boolean] :symbolize_keys (false) whether to return the keys as symbols or strings - # @return [Hash] the Ruby data structure represented by the input - def self.load_file(path, **options) - # By default Ruby sets the external encoding of an IO object to the - # default external encoding. The default external encoding is set by - # locale encoding or the interpreter -E option. - tmp = File.read(path, :encoding => "utf-8") - Tomlrb.parse(tmp, options) - end -end diff --git a/source/toml-parser/tomlrb/generated_parser.rb b/source/toml-parser/tomlrb/generated_parser.rb deleted file mode 100644 index ebf815e7d..000000000 --- a/source/toml-parser/tomlrb/generated_parser.rb +++ /dev/null @@ -1,542 +0,0 @@ -# -# DO NOT MODIFY!!!! -# This file is automatically generated by Racc 1.4.14 -# from Racc grammer file "". -# - -require 'racc/parser.rb' -module Tomlrb - class GeneratedParser < Racc::Parser -##### State transition tables begin ### - -racc_action_table = [ - 2, 17, 11, 31, 12, 31, 13, 27, 14, 77, - 15, 16, 8, 78, 32, 10, 33, 29, 34, 29, - 57, 58, 59, 60, 56, 53, 52, 54, 55, 46, - 40, 41, 10, 57, 58, 59, 60, 56, 53, 52, - 54, 55, 46, 69, 70, 10, 57, 58, 59, 60, - 56, 53, 52, 54, 55, 46, 35, 36, 10, 57, - 58, 59, 60, 56, 53, 52, 54, 55, 46, 37, - 38, 10, 57, 58, 59, 60, 56, 53, 52, 54, - 55, 46, 43, 66, 10, 57, 58, 59, 60, 56, - 53, 52, 54, 55, 46, nil, nil, 10, 57, 58, - 59, 60, 56, 53, 52, 54, 55, 46, nil, nil, - 10, 57, 58, 59, 60, 56, 53, 52, 54, 55, - 46, 73, nil, 10, 57, 58, 59, 60, 56, 53, - 52, 54, 55, 46, 73, 21, 10, 22, nil, 23, - nil, 24, nil, 25, 26, 21, 19, 22, nil, 23, - nil, 24, nil, 25, 26, nil, 19 ] - -racc_action_check = [ - 1, 2, 1, 9, 1, 70, 1, 8, 1, 74, - 1, 1, 1, 74, 11, 1, 12, 9, 13, 70, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 20, 20, 32, 33, 33, 33, 33, 33, 33, 33, - 33, 33, 33, 42, 42, 33, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 14, 15, 34, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 16, - 19, 35, 36, 36, 36, 36, 36, 36, 36, 36, - 36, 36, 30, 40, 36, 37, 37, 37, 37, 37, - 37, 37, 37, 37, 37, nil, nil, 37, 43, 43, - 43, 43, 43, 43, 43, 43, 43, 43, nil, nil, - 43, 45, 45, 45, 45, 45, 45, 45, 45, 45, - 45, 45, nil, 45, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 7, 78, 7, nil, 7, - nil, 7, nil, 7, 7, 41, 7, 41, nil, 41, - nil, 41, nil, 41, 41, nil, 41 ] - -racc_action_pointer = [ - nil, 0, 1, nil, nil, nil, nil, 133, -5, 1, - nil, -4, -2, 0, 38, 39, 51, nil, nil, 57, - 17, nil, nil, nil, nil, nil, nil, nil, nil, nil, - 64, nil, 17, 30, 43, 56, 69, 82, nil, nil, - 70, 143, 27, 95, nil, 108, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - 3, nil, nil, nil, -4, nil, nil, nil, 121, nil ] - -racc_action_default = [ - -1, -56, -56, -2, -3, -4, -5, -56, -8, -56, - -22, -56, -56, -56, -56, -56, -56, 80, -6, -10, - -56, -15, -16, -17, -18, -19, -20, -7, -21, -23, - -56, -27, -46, -46, -46, -46, -46, -46, -9, -11, - -13, -56, -56, -46, -29, -46, -40, -41, -42, -43, - -44, -45, -47, -48, -49, -50, -51, -52, -53, -54, - -55, -30, -31, -32, -33, -34, -12, -14, -24, -25, - -56, -28, -35, -36, -56, -26, -37, -38, -46, -39 ] - -racc_goto_table = [ - 28, 18, 1, 72, 44, 61, 62, 63, 64, 65, - 3, 4, 5, 6, 7, 71, 39, 42, 68, 76, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, 67, 79, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 75 ] - -racc_goto_check = [ - 11, 7, 1, 18, 15, 15, 15, 15, 15, 15, - 2, 3, 4, 5, 6, 15, 9, 13, 14, 19, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, 7, 18, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 11 ] - -racc_goto_pointer = [ - nil, 2, 9, 10, 11, 12, 13, -6, nil, -4, - nil, -9, nil, -13, -24, -28, nil, nil, -42, -55, - nil, nil, nil ] - -racc_goto_default = [ - nil, nil, nil, nil, nil, 49, nil, nil, 20, nil, - 9, nil, 30, nil, nil, 74, 48, 45, nil, nil, - 47, 50, 51 ] - -racc_reduce_table = [ - 0, 0, :racc_error, - 0, 20, :_reduce_none, - 2, 20, :_reduce_none, - 1, 21, :_reduce_none, - 1, 21, :_reduce_none, - 1, 21, :_reduce_none, - 2, 22, :_reduce_none, - 2, 25, :_reduce_7, - 1, 25, :_reduce_8, - 2, 26, :_reduce_9, - 1, 26, :_reduce_10, - 2, 26, :_reduce_none, - 2, 28, :_reduce_12, - 1, 28, :_reduce_13, - 2, 28, :_reduce_none, - 1, 27, :_reduce_15, - 1, 27, :_reduce_16, - 1, 27, :_reduce_17, - 1, 27, :_reduce_18, - 1, 27, :_reduce_19, - 1, 27, :_reduce_20, - 2, 24, :_reduce_none, - 1, 29, :_reduce_22, - 1, 30, :_reduce_23, - 3, 30, :_reduce_none, - 1, 33, :_reduce_25, - 2, 33, :_reduce_none, - 1, 31, :_reduce_27, - 2, 32, :_reduce_none, - 3, 23, :_reduce_29, - 3, 23, :_reduce_30, - 3, 23, :_reduce_31, - 3, 23, :_reduce_32, - 3, 23, :_reduce_33, - 3, 23, :_reduce_34, - 2, 35, :_reduce_none, - 1, 37, :_reduce_36, - 2, 37, :_reduce_none, - 1, 38, :_reduce_38, - 2, 38, :_reduce_none, - 1, 36, :_reduce_40, - 1, 34, :_reduce_41, - 1, 34, :_reduce_none, - 1, 34, :_reduce_none, - 1, 39, :_reduce_none, - 1, 39, :_reduce_none, - 0, 41, :_reduce_none, - 1, 41, :_reduce_47, - 1, 41, :_reduce_48, - 1, 41, :_reduce_49, - 1, 41, :_reduce_50, - 1, 41, :_reduce_51, - 1, 40, :_reduce_52, - 1, 40, :_reduce_53, - 1, 40, :_reduce_54, - 1, 40, :_reduce_55 ] - -racc_reduce_n = 56 - -racc_shift_n = 80 - -racc_token_table = { - false => 0, - :error => 1, - :IDENTIFIER => 2, - :STRING_MULTI => 3, - :STRING_BASIC => 4, - :STRING_LITERAL_MULTI => 5, - :STRING_LITERAL => 6, - :DATETIME => 7, - :INTEGER => 8, - :FLOAT => 9, - :TRUE => 10, - :FALSE => 11, - "[" => 12, - "]" => 13, - "." => 14, - "{" => 15, - "}" => 16, - "," => 17, - "=" => 18 } - -racc_nt_base = 19 - -racc_use_result_var = true - -Racc_arg = [ - racc_action_table, - racc_action_check, - racc_action_default, - racc_action_pointer, - racc_goto_table, - racc_goto_check, - racc_goto_default, - racc_goto_pointer, - racc_nt_base, - racc_reduce_table, - racc_token_table, - racc_shift_n, - racc_reduce_n, - racc_use_result_var ] - -Racc_token_to_s_table = [ - "$end", - "error", - "IDENTIFIER", - "STRING_MULTI", - "STRING_BASIC", - "STRING_LITERAL_MULTI", - "STRING_LITERAL", - "DATETIME", - "INTEGER", - "FLOAT", - "TRUE", - "FALSE", - "\"[\"", - "\"]\"", - "\".\"", - "\"{\"", - "\"}\"", - "\",\"", - "\"=\"", - "$start", - "expressions", - "expression", - "table", - "assignment", - "inline_table", - "table_start", - "table_continued", - "table_identifier", - "table_next", - "inline_table_start", - "inline_continued", - "inline_assignment_key", - "inline_assignment_value", - "inline_next", - "value", - "array", - "start_array", - "array_continued", - "array_next", - "scalar", - "string", - "literal" ] - -Racc_debug_parser = false - -##### State transition tables end ##### - -# reduce 0 omitted - -# reduce 1 omitted - -# reduce 2 omitted - -# reduce 3 omitted - -# reduce 4 omitted - -# reduce 5 omitted - -# reduce 6 omitted - -module_eval(<<'.,.,', 'parser.y', 15) - def _reduce_7(val, _values, result) - @handler.start_(:array_of_tables) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 16) - def _reduce_8(val, _values, result) - @handler.start_(:table) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 19) - def _reduce_9(val, _values, result) - array = @handler.end_(:array_of_tables); @handler.set_context(array, is_array_of_tables: true) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 20) - def _reduce_10(val, _values, result) - array = @handler.end_(:table); @handler.set_context(array) - result - end -.,., - -# reduce 11 omitted - -module_eval(<<'.,.,', 'parser.y', 24) - def _reduce_12(val, _values, result) - array = @handler.end_(:array_of_tables); @handler.set_context(array, is_array_of_tables: true) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 25) - def _reduce_13(val, _values, result) - array = @handler.end_(:table); @handler.set_context(array) - result - end -.,., - -# reduce 14 omitted - -module_eval(<<'.,.,', 'parser.y', 29) - def _reduce_15(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 30) - def _reduce_16(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 31) - def _reduce_17(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 32) - def _reduce_18(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 33) - def _reduce_19(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 34) - def _reduce_20(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -# reduce 21 omitted - -module_eval(<<'.,.,', 'parser.y', 40) - def _reduce_22(val, _values, result) - @handler.start_(:inline) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 43) - def _reduce_23(val, _values, result) - array = @handler.end_(:inline); @handler.push(Hash[*array]) - result - end -.,., - -# reduce 24 omitted - -module_eval(<<'.,.,', 'parser.y', 48) - def _reduce_25(val, _values, result) - array = @handler.end_(:inline) - array.map!.with_index{ |n,i| i.even? ? n.to_sym : n } if @handler.symbolize_keys - @handler.push(Hash[*array]) - - result - end -.,., - -# reduce 26 omitted - -module_eval(<<'.,.,', 'parser.y', 55) - def _reduce_27(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -# reduce 28 omitted - -module_eval(<<'.,.,', 'parser.y', 61) - def _reduce_29(val, _values, result) - @handler.assign(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 62) - def _reduce_30(val, _values, result) - @handler.assign(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 63) - def _reduce_31(val, _values, result) - @handler.assign(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 64) - def _reduce_32(val, _values, result) - @handler.assign(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 65) - def _reduce_33(val, _values, result) - @handler.assign(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 66) - def _reduce_34(val, _values, result) - @handler.assign(val[0]) - result - end -.,., - -# reduce 35 omitted - -module_eval(<<'.,.,', 'parser.y', 72) - def _reduce_36(val, _values, result) - array = @handler.end_(:array); @handler.push(array) - result - end -.,., - -# reduce 37 omitted - -module_eval(<<'.,.,', 'parser.y', 76) - def _reduce_38(val, _values, result) - array = @handler.end_(:array); @handler.push(array) - result - end -.,., - -# reduce 39 omitted - -module_eval(<<'.,.,', 'parser.y', 80) - def _reduce_40(val, _values, result) - @handler.start_(:array) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 83) - def _reduce_41(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -# reduce 42 omitted - -# reduce 43 omitted - -# reduce 44 omitted - -# reduce 45 omitted - -# reduce 46 omitted - -module_eval(<<'.,.,', 'parser.y', 92) - def _reduce_47(val, _values, result) - result = val[0].to_f - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 93) - def _reduce_48(val, _values, result) - result = val[0].to_i - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 94) - def _reduce_49(val, _values, result) - result = true - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 95) - def _reduce_50(val, _values, result) - result = false - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 96) - def _reduce_51(val, _values, result) - result = Time.new(*val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 99) - def _reduce_52(val, _values, result) - result = StringUtils.replace_escaped_chars(StringUtils.multiline_replacements(val[0])) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 100) - def _reduce_53(val, _values, result) - result = StringUtils.replace_escaped_chars(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 101) - def _reduce_54(val, _values, result) - result = StringUtils.strip_spaces(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 102) - def _reduce_55(val, _values, result) - result = val[0] - result - end -.,., - -def _reduce_none(val, _values, result) - val[0] -end - - end # class GeneratedParser - end # module Tomlrb diff --git a/source/toml-parser/tomlrb/handler.rb b/source/toml-parser/tomlrb/handler.rb deleted file mode 100644 index d60b54bc3..000000000 --- a/source/toml-parser/tomlrb/handler.rb +++ /dev/null @@ -1,73 +0,0 @@ -module Tomlrb - class Handler - attr_reader :output, :symbolize_keys - - def initialize(**options) - @output = {} - @current = @output - @stack = [] - @array_names = [] - @symbolize_keys = options[:symbolize_keys] - end - - def set_context(identifiers, is_array_of_tables: false) - @current = @output - - deal_with_array_of_tables(identifiers, is_array_of_tables) do |identifierz| - identifierz.each do |k| - k = k.to_sym if @symbolize_keys - if @current[k].is_a?(Array) - @current[k] << {} if @current[k].empty? - @current = @current[k].last - else - @current[k] ||= {} - @current = @current[k] - end - end - end - end - - def deal_with_array_of_tables(identifiers, is_array_of_tables) - identifiers.map!{|n| n.gsub("\"", '')} - stringified_identifier = identifiers.join('.') - - if is_array_of_tables - @array_names << stringified_identifier - last_identifier = identifiers.pop - elsif @array_names.include?(stringified_identifier) - raise ParseError, 'Cannot define a normal table with the same name as an already established array' - end - - yield(identifiers) - - if is_array_of_tables - last_identifier = last_identifier.to_sym if @symbolize_keys - @current[last_identifier] ||= [] - @current[last_identifier] << {} - @current = @current[last_identifier].last - end - end - - def assign(k) - k = k.to_sym if @symbolize_keys - @current[k] = @stack.pop - end - - def push(o) - @stack << o - end - - def start_(type) - push([type]) - end - - def end_(type) - array = [] - while (value = @stack.pop) != [type] - raise ParseError, 'Unclosed table' unless value - array.unshift(value) - end - array - end - end -end diff --git a/source/toml-parser/tomlrb/parser.rb b/source/toml-parser/tomlrb/parser.rb deleted file mode 100644 index 31771a1ca..000000000 --- a/source/toml-parser/tomlrb/parser.rb +++ /dev/null @@ -1,18 +0,0 @@ -require_relative "generated_parser" - -class Tomlrb::Parser < Tomlrb::GeneratedParser - def initialize(tokenizer, **options) - @tokenizer = tokenizer - @handler = Tomlrb::Handler.new(options) - super() - end - - def next_token - @tokenizer.next_token - end - - def parse - do_parse - @handler - end -end diff --git a/source/toml-parser/tomlrb/parser.y b/source/toml-parser/tomlrb/parser.y deleted file mode 100644 index fcebcac06..000000000 --- a/source/toml-parser/tomlrb/parser.y +++ /dev/null @@ -1,104 +0,0 @@ -class Tomlrb::GeneratedParser -token IDENTIFIER STRING_MULTI STRING_BASIC STRING_LITERAL_MULTI STRING_LITERAL DATETIME INTEGER FLOAT TRUE FALSE -rule - expressions - | expressions expression - ; - expression - : table - | assignment - | inline_table - ; - table - : table_start table_continued - ; - table_start - : '[' '[' { @handler.start_(:array_of_tables) } - | '[' { @handler.start_(:table) } - ; - table_continued - : ']' ']' { array = @handler.end_(:array_of_tables); @handler.set_context(array, is_array_of_tables: true) } - | ']' { array = @handler.end_(:table); @handler.set_context(array) } - | table_identifier table_next - ; - table_next - : ']' ']' { array = @handler.end_(:array_of_tables); @handler.set_context(array, is_array_of_tables: true) } - | ']' { array = @handler.end_(:table); @handler.set_context(array) } - | '.' table_continued - ; - table_identifier - : IDENTIFIER { @handler.push(val[0]) } - | STRING_BASIC { @handler.push(val[0]) } - | STRING_LITERAL { @handler.push(val[0]) } - | INTEGER { @handler.push(val[0]) } - | TRUE { @handler.push(val[0]) } - | FALSE { @handler.push(val[0]) } - ; - inline_table - : inline_table_start inline_continued - ; - inline_table_start - : '{' { @handler.start_(:inline) } - ; - inline_continued - : '}' { array = @handler.end_(:inline); @handler.push(Hash[*array]) } - | inline_assignment_key inline_assignment_value inline_next - ; - inline_next - : '}' { - array = @handler.end_(:inline) - array.map!.with_index{ |n,i| i.even? ? n.to_sym : n } if @handler.symbolize_keys - @handler.push(Hash[*array]) - } - | ',' inline_continued - ; - inline_assignment_key - : IDENTIFIER { @handler.push(val[0]) } - ; - inline_assignment_value - : '=' value - ; - assignment - : IDENTIFIER '=' value { @handler.assign(val[0]) } - | STRING_BASIC '=' value { @handler.assign(val[0]) } - | STRING_LITERAL '=' value { @handler.assign(val[0]) } - | INTEGER '=' value { @handler.assign(val[0]) } - | TRUE '=' value { @handler.assign(val[0]) } - | FALSE '=' value { @handler.assign(val[0]) } - ; - array - : start_array array_continued - ; - array_continued - : ']' { array = @handler.end_(:array); @handler.push(array) } - | value array_next - ; - array_next - : ']' { array = @handler.end_(:array); @handler.push(array) } - | ',' array_continued - ; - start_array - : '[' { @handler.start_(:array) } - ; - value - : scalar { @handler.push(val[0]) } - | array - | inline_table - ; - scalar - : string - | literal - ; - literal - | FLOAT { result = val[0].to_f } - | INTEGER { result = val[0].to_i } - | TRUE { result = true } - | FALSE { result = false } - | DATETIME { result = Time.new(*val[0])} - ; - string - : STRING_MULTI { result = StringUtils.replace_escaped_chars(StringUtils.multiline_replacements(val[0])) } - | STRING_BASIC { result = StringUtils.replace_escaped_chars(val[0]) } - | STRING_LITERAL_MULTI { result = StringUtils.strip_spaces(val[0]) } - | STRING_LITERAL { result = val[0] } - ; diff --git a/source/toml-parser/tomlrb/scanner.rb b/source/toml-parser/tomlrb/scanner.rb deleted file mode 100644 index d0f479eef..000000000 --- a/source/toml-parser/tomlrb/scanner.rb +++ /dev/null @@ -1,54 +0,0 @@ -require 'strscan' - -module Tomlrb - class Scanner - COMMENT = /#.*/ - IDENTIFIER = /[A-Za-z0-9_-]+/ - SPACE = /[ \t\r\n]/ - STRING_BASIC = /(["])(?:\\?.)*?\1/ - STRING_MULTI = /"{3}([\s\S]*?"{3,4})/m - STRING_LITERAL = /(['])(?:\\?.)*?\1/ - STRING_LITERAL_MULTI = /'{3}([\s\S]*?'{3})/m - DATETIME = /(-?\d{4})-(\d{2})-(\d{2})(?:(?:t|\s)(\d{2}):(\d{2}):(\d{2}(?:\.\d+)?))?(z|[-+]\d{2}:\d{2})?/i - FLOAT = /[+-]?(?:[0-9_]+\.[0-9_]*|\d+(?=[eE]))(?:[eE][+-]?[0-9_]+)?/ - INTEGER = /[+-]?([1-9](_?\d)*|0)(?![A-Za-z0-9_-]+)/ - TRUE = /true/ - FALSE = /false/ - - def initialize(io) - @ss = StringScanner.new(io.read) - end - - def next_token - return if @ss.eos? - - case - when @ss.scan(SPACE) then next_token - when @ss.scan(COMMENT) then next_token - when @ss.scan(DATETIME) then process_datetime - when text = @ss.scan(STRING_MULTI) then [:STRING_MULTI, text[3..-4]] - when text = @ss.scan(STRING_BASIC) then [:STRING_BASIC, text[1..-2]] - when text = @ss.scan(STRING_LITERAL_MULTI) then [:STRING_LITERAL_MULTI, text[3..-4]] - when text = @ss.scan(STRING_LITERAL) then [:STRING_LITERAL, text[1..-2]] - when text = @ss.scan(FLOAT) then [:FLOAT, text] - when text = @ss.scan(INTEGER) then [:INTEGER, text] - when text = @ss.scan(TRUE) then [:TRUE, text] - when text = @ss.scan(FALSE) then [:FALSE, text] - when text = @ss.scan(IDENTIFIER) then [:IDENTIFIER, text] - else - x = @ss.getch - [x, x] - end - end - - def process_datetime - if @ss[7].nil? - offset = '+00:00' - else - offset = @ss[7].gsub('Z', '+00:00') - end - args = [@ss[1], @ss[2], @ss[3], @ss[4] || 0, @ss[5] || 0, @ss[6].to_f, offset] - [:DATETIME, args] - end - end -end diff --git a/source/toml-parser/tomlrb/string_utils.rb b/source/toml-parser/tomlrb/string_utils.rb deleted file mode 100644 index 53d27e414..000000000 --- a/source/toml-parser/tomlrb/string_utils.rb +++ /dev/null @@ -1,33 +0,0 @@ -module Tomlrb - class StringUtils - - SPECIAL_CHARS = { - '\\t' => "\t", - '\\b' => "\b", - '\\f' => "\f", - '\\n' => "\n", - '\\r' => "\r", - '\\"' => '"', - '\\\\' => '\\' - }.freeze - - def self.multiline_replacements(str) - strip_spaces(str).gsub(/\\\n\s+/, '') - end - - def self.replace_escaped_chars(str) - str.gsub(/\\(u[\da-fA-F]{4}|U[\da-fA-F]{8}|.)/) do |m| - if m.size == 2 - SPECIAL_CHARS[m] || (raise Tomlrb::ParseError.new "Escape sequence #{m} is reserved") - else - m[2..-1].to_i(16).chr(Encoding::UTF_8) - end - end - end - - def self.strip_spaces(str) - str[0] = '' if str[0] == "\n" - str - end - end -end diff --git a/source/toml-parser/tomlrb/version.rb b/source/toml-parser/tomlrb/version.rb deleted file mode 100644 index b72a81b60..000000000 --- a/source/toml-parser/tomlrb/version.rb +++ /dev/null @@ -1,3 +0,0 @@ -module Tomlrb - VERSION = "1.2.8" -end From 9eaa3e44ec475cb132527be425089f1adc8b0ab9 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Wed, 4 May 2022 23:19:08 +0000 Subject: [PATCH 14/36] add ganga's changes --- .pipelines/azure_pipeline_dev.yaml | 32 +----------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/.pipelines/azure_pipeline_dev.yaml b/.pipelines/azure_pipeline_dev.yaml index c4723e8e1..a038446a6 100644 --- a/.pipelines/azure_pipeline_dev.yaml +++ b/.pipelines/azure_pipeline_dev.yaml @@ -183,37 +183,7 @@ jobs: docker push ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion) } -- job: build_windows_2022 - dependsOn: - - common - pool: - name: Azure-Pipelines-Windows-CI-Test-EO - variables: - windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] - windows2022BaseImageVersion: ltsc2022 - steps: - - task: PowerShell@2 - inputs: - targetType: 'filePath' - filePath: $(System.DefaultWorkingDirectory)/scripts/build/windows/install-build-pre-requisites.ps1 - displayName: 'install prereqs' - - - script: | - setlocal enabledelayedexpansion - powershell.exe -ExecutionPolicy Unrestricted -NoProfile -WindowStyle Hidden -File "build\windows\Makefile.ps1" - endlocal - exit /B %ERRORLEVEL% - displayName: 'build base' - - - task: AzureCLI@2 - displayName: "Docker windows build for ltsc2022" - inputs: - azureSubscription: ${{ variables.armServiceConnectionName }} - scriptType: ps - scriptLocation: inlineScript - inlineScript: | - mkdir -p $(Build.ArtifactStagingDirectory)/windows - cd kubernetes/windows + docker build --isolation=hyperv --tag ${{ variables.repoImageName }}:$(windowsImageTag) --build-arg IMAGE_TAG=$(windowsImageTag) . az --version az account show From 7253451caff759703cfcd178ac01d89264fd5d0e Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Thu, 5 May 2022 17:52:22 +0000 Subject: [PATCH 15/36] fix bug --- kubernetes/linux/Dockerfile.multiarch | 2 +- source/plugins/ruby/kubelet_utils.rb | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/kubernetes/linux/Dockerfile.multiarch b/kubernetes/linux/Dockerfile.multiarch index f5b4cdd60..38103dd65 100644 --- a/kubernetes/linux/Dockerfile.multiarch +++ b/kubernetes/linux/Dockerfile.multiarch @@ -24,7 +24,7 @@ ENV AZMON_COLLECT_ENV False ENV KUBE_CLIENT_BACKOFF_BASE 1 ENV KUBE_CLIENT_BACKOFF_DURATION 0 ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9 -RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg2 make && rm -rf /var/lib/apt/lists/* +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg make && rm -rf /var/lib/apt/lists/* COPY --from=builder /src/kubernetes/linux/Linux_ULINUX_1.0_*_64_Release/docker-cimprov-*.*.*-*.*.sh $tmpdir/ COPY kubernetes/linux/setup.sh kubernetes/linux/main.sh kubernetes/linux/defaultpromenvvariables kubernetes/linux/defaultpromenvvariables-rs kubernetes/linux/defaultpromenvvariables-sidecar kubernetes/linux/mdsd.xml kubernetes/linux/envmdsd kubernetes/linux/logrotate.conf $tmpdir/ diff --git a/source/plugins/ruby/kubelet_utils.rb b/source/plugins/ruby/kubelet_utils.rb index 368ca8639..22d079fa1 100644 --- a/source/plugins/ruby/kubelet_utils.rb +++ b/source/plugins/ruby/kubelet_utils.rb @@ -80,7 +80,11 @@ def get_node_allocatable(cpu_capacity, memory_capacity) ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::kubereserved_memory: #{errorStr}") end begin - systemReserved_cpu = parsed_response["kubeletconfig"]["systemReserved"]["cpu"] + if (parsed_response["kubeletconfig"]["systemReserved"].has_key?("cpu")) + systemReserved_cpu = parsed_response["kubeletconfig"]["systemReserved"]["cpu"] + else + systemReserved_cpu = nil + end if systemReserved_cpu.nil? || systemReserved_cpu == "" systemReserved_cpu = "0.0" end @@ -106,7 +110,11 @@ def get_node_allocatable(cpu_capacity, memory_capacity) end begin - systemReserved_memory = parsed_response["kubeletconfig"]["systemReserved"]["memory"] + if (parsed_response["kubeletconfig"]["systemReserved"].has_key?("memory")) + systemReserved_memory = parsed_response["kubeletconfig"]["systemReserved"]["memory"] + else + systemReserved_memory = nil + end if systemReserved_memory.nil? || systemReserved_memory == "" systemReserved_memory = "0.0" end From 4849f729075c59dbee81615e9f24386663398716 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Thu, 5 May 2022 19:48:41 +0000 Subject: [PATCH 16/36] revert changes --- source/plugins/ruby/kubelet_utils.rb | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/source/plugins/ruby/kubelet_utils.rb b/source/plugins/ruby/kubelet_utils.rb index 22d079fa1..368ca8639 100644 --- a/source/plugins/ruby/kubelet_utils.rb +++ b/source/plugins/ruby/kubelet_utils.rb @@ -80,11 +80,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::kubereserved_memory: #{errorStr}") end begin - if (parsed_response["kubeletconfig"]["systemReserved"].has_key?("cpu")) - systemReserved_cpu = parsed_response["kubeletconfig"]["systemReserved"]["cpu"] - else - systemReserved_cpu = nil - end + systemReserved_cpu = parsed_response["kubeletconfig"]["systemReserved"]["cpu"] if systemReserved_cpu.nil? || systemReserved_cpu == "" systemReserved_cpu = "0.0" end @@ -110,11 +106,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) end begin - if (parsed_response["kubeletconfig"]["systemReserved"].has_key?("memory")) - systemReserved_memory = parsed_response["kubeletconfig"]["systemReserved"]["memory"] - else - systemReserved_memory = nil - end + systemReserved_memory = parsed_response["kubeletconfig"]["systemReserved"]["memory"] if systemReserved_memory.nil? || systemReserved_memory == "" systemReserved_memory = "0.0" end From b59c3f61381f5fff1718885f6a5e9c45ada0bf10 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Mon, 9 May 2022 21:31:17 +0000 Subject: [PATCH 17/36] address warning --- build/windows/installer/conf/fluent.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/windows/installer/conf/fluent.conf b/build/windows/installer/conf/fluent.conf index a78ac58fa..73d62a3ff 100644 --- a/build/windows/installer/conf/fluent.conf +++ b/build/windows/installer/conf/fluent.conf @@ -1,5 +1,5 @@ - type heartbeat_request + @type heartbeat_request run_interval 30m @log_level info From 169d2c2f9ec45e27a8cdea5770b9adb491d47f31 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 10 May 2022 16:21:09 +0000 Subject: [PATCH 18/36] revert changes in dev pipeline --- .pipelines/azure_pipeline_dev.yaml | 32 +++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.pipelines/azure_pipeline_dev.yaml b/.pipelines/azure_pipeline_dev.yaml index a038446a6..c4723e8e1 100644 --- a/.pipelines/azure_pipeline_dev.yaml +++ b/.pipelines/azure_pipeline_dev.yaml @@ -183,7 +183,37 @@ jobs: docker push ${{ variables.repoImageName }}:$(windowsImageTag)-$(windows2019BaseImageVersion) } - docker build --isolation=hyperv --tag ${{ variables.repoImageName }}:$(windowsImageTag) --build-arg IMAGE_TAG=$(windowsImageTag) . +- job: build_windows_2022 + dependsOn: + - common + pool: + name: Azure-Pipelines-Windows-CI-Test-EO + variables: + windowsImageTag: $[ dependencies.common.outputs['setup.windowsImageTag'] ] + windows2022BaseImageVersion: ltsc2022 + steps: + - task: PowerShell@2 + inputs: + targetType: 'filePath' + filePath: $(System.DefaultWorkingDirectory)/scripts/build/windows/install-build-pre-requisites.ps1 + displayName: 'install prereqs' + + - script: | + setlocal enabledelayedexpansion + powershell.exe -ExecutionPolicy Unrestricted -NoProfile -WindowStyle Hidden -File "build\windows\Makefile.ps1" + endlocal + exit /B %ERRORLEVEL% + displayName: 'build base' + + - task: AzureCLI@2 + displayName: "Docker windows build for ltsc2022" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: ps + scriptLocation: inlineScript + inlineScript: | + mkdir -p $(Build.ArtifactStagingDirectory)/windows + cd kubernetes/windows az --version az account show From b1ee385ffaa845156d81ecbc08e7ba220e140ba5 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 18 May 2022 20:36:44 -0700 Subject: [PATCH 19/36] builtin json inplace of yajl --- .../ruby/ApplicationInsightsUtility.rb | 2 +- .../plugins/ruby/CAdvisorMetricsAPIClient.rb | 2 +- .../plugins/ruby/ContainerInventoryState.rb | 2 +- source/plugins/ruby/DockerApiClient.rb | 2 +- source/plugins/ruby/KubernetesApiClient.rb | 4 ++-- source/plugins/ruby/MdmMetricsGenerator.rb | 23 +++++++++---------- .../plugins/ruby/arc_k8s_cluster_identity.rb | 2 +- source/plugins/ruby/filter_cadvisor2mdm.rb | 2 +- source/plugins/ruby/filter_inventory2mdm.rb | 2 +- source/plugins/ruby/filter_telegraf2mdm.rb | 2 +- source/plugins/ruby/in_cadvisor_perf.rb | 2 +- source/plugins/ruby/in_containerinventory.rb | 2 +- source/plugins/ruby/in_kube_events.rb | 3 +-- source/plugins/ruby/in_kube_nodes.rb | 3 +-- source/plugins/ruby/in_kube_podinventory.rb | 5 ++-- source/plugins/ruby/in_kube_pvinventory.rb | 3 +-- .../plugins/ruby/in_kubestate_deployments.rb | 3 +-- source/plugins/ruby/in_kubestate_hpa.rb | 3 +-- source/plugins/ruby/in_win_cadvisor_perf.rb | 2 +- source/plugins/ruby/kubelet_utils.rb | 4 ++-- .../ruby/kubernetes_container_inventory.rb | 3 +-- .../channel/contracts/json_serializable.rb | 2 +- .../channel/sender_base.rb | 2 +- source/plugins/ruby/out_mdm.rb | 2 +- source/plugins/ruby/podinventory_to_mdm.rb | 2 +- 25 files changed, 38 insertions(+), 46 deletions(-) diff --git a/source/plugins/ruby/ApplicationInsightsUtility.rb b/source/plugins/ruby/ApplicationInsightsUtility.rb index 70d0a400e..b34cb20ee 100644 --- a/source/plugins/ruby/ApplicationInsightsUtility.rb +++ b/source/plugins/ruby/ApplicationInsightsUtility.rb @@ -7,7 +7,7 @@ class ApplicationInsightsUtility require_relative "DockerApiClient" require_relative "oms_common" require_relative "proxy_utils" - require "yajl/json_gem" + require "json" require "base64" @@HeartBeat = "HeartBeatEvent" diff --git a/source/plugins/ruby/CAdvisorMetricsAPIClient.rb b/source/plugins/ruby/CAdvisorMetricsAPIClient.rb index 20faf4619..521a7f7c1 100644 --- a/source/plugins/ruby/CAdvisorMetricsAPIClient.rb +++ b/source/plugins/ruby/CAdvisorMetricsAPIClient.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true class CAdvisorMetricsAPIClient - require "yajl/json_gem" + require "json" require "logger" require "net/http" require "net/https" diff --git a/source/plugins/ruby/ContainerInventoryState.rb b/source/plugins/ruby/ContainerInventoryState.rb index 170fa65e3..7e5ca18e8 100644 --- a/source/plugins/ruby/ContainerInventoryState.rb +++ b/source/plugins/ruby/ContainerInventoryState.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true class ContainerInventoryState - require 'yajl/json_gem' + require 'json' require_relative 'omslog' @@InventoryDirectory = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory/" diff --git a/source/plugins/ruby/DockerApiClient.rb b/source/plugins/ruby/DockerApiClient.rb index 53dd1f39f..cff9f359f 100644 --- a/source/plugins/ruby/DockerApiClient.rb +++ b/source/plugins/ruby/DockerApiClient.rb @@ -3,7 +3,7 @@ class DockerApiClient require "socket" - require "yajl/json_gem" + require "json" require "timeout" require_relative "omslog" require_relative "DockerApiRestHelper" diff --git a/source/plugins/ruby/KubernetesApiClient.rb b/source/plugins/ruby/KubernetesApiClient.rb index 8925248d7..392f67263 100644 --- a/source/plugins/ruby/KubernetesApiClient.rb +++ b/source/plugins/ruby/KubernetesApiClient.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true class KubernetesApiClient - require "yajl/json_gem" + require "json" require "logger" require "net/http" require "net/https" @@ -763,7 +763,7 @@ def getResourcesAndContinuationToken(uri, api_group: nil) @Log.info "KubernetesApiClient::getResourcesAndContinuationToken : Done getting resources from Kube API using url: #{uri} @ #{Time.now.utc.iso8601}" if !resourceInfo.nil? @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:Start:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" - resourceInventory = Yajl::Parser.parse(StringIO.new(resourceInfo.body)) + resourceInventory = JSON.parse(StringIO.new(resourceInfo.body)) @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:End:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" resourceInfo = nil end diff --git a/source/plugins/ruby/MdmMetricsGenerator.rb b/source/plugins/ruby/MdmMetricsGenerator.rb index f4904697c..cfd2404db 100644 --- a/source/plugins/ruby/MdmMetricsGenerator.rb +++ b/source/plugins/ruby/MdmMetricsGenerator.rb @@ -3,8 +3,7 @@ class MdmMetricsGenerator require "logger" - require "yajl/json_gem" - require "json" + require "json" require_relative "MdmAlertTemplates" require_relative "ApplicationInsightsUtility" require_relative "constants" @@ -140,7 +139,7 @@ def appendPodMetrics(records, metricName, metricHash, batch_time, metricsTemplat containerCountMetricValue: value, } end - records.push(Yajl::Parser.parse(StringIO.new(record))) + records.push(JSON.parse(StringIO.new(record))) } else @log.info "No records found in hash for metric: #{metricName}" @@ -334,7 +333,7 @@ def getContainerResourceUtilMetricRecords(recordTimeStamp, metricName, percentag containerResourceUtilizationPercentage: percentageMetricValue, thresholdPercentageDimValue: thresholdPercentage, } - records.push(Yajl::Parser.parse(StringIO.new(resourceUtilRecord))) + records.push(JSON.parse(StringIO.new(resourceUtilRecord))) # Adding another metric for threshold violation resourceThresholdViolatedRecord = MdmAlertTemplates::Container_resource_threshold_violation_template % { @@ -347,7 +346,7 @@ def getContainerResourceUtilMetricRecords(recordTimeStamp, metricName, percentag containerResourceThresholdViolated: isZeroFill ? 0 : 1, thresholdPercentageDimValue: thresholdPercentage, } - records.push(Yajl::Parser.parse(StringIO.new(resourceThresholdViolatedRecord))) + records.push(JSON.parse(StringIO.new(resourceThresholdViolatedRecord))) rescue => errorStr @log.info "Error in getContainerResourceUtilMetricRecords: #{errorStr}" ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) @@ -374,7 +373,7 @@ def getPVResourceUtilMetricRecords(recordTimeStamp, metricName, computer, percen pvResourceUtilizationPercentage: percentageMetricValue, thresholdPercentageDimValue: thresholdPercentage, } - records.push(Yajl::Parser.parse(StringIO.new(resourceUtilRecord))) + records.push(JSON.parse(StringIO.new(resourceUtilRecord))) # Adding another metric for threshold violation resourceThresholdViolatedRecord = MdmAlertTemplates::PV_resource_threshold_violation_template % { @@ -387,7 +386,7 @@ def getPVResourceUtilMetricRecords(recordTimeStamp, metricName, computer, percen pvResourceThresholdViolated: isZeroFill ? 0 : 1, thresholdPercentageDimValue: thresholdPercentage, } - records.push(Yajl::Parser.parse(StringIO.new(resourceThresholdViolatedRecord))) + records.push(JSON.parse(StringIO.new(resourceThresholdViolatedRecord))) rescue => errorStr @log.info "Error in getPVResourceUtilMetricRecords: #{errorStr}" ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) @@ -418,7 +417,7 @@ def getDiskUsageMetricRecords(record) devicevalue: deviceName, diskUsagePercentageValue: usedPercent, } - records.push(Yajl::Parser.parse(StringIO.new(diskUsedPercentageRecord))) + records.push(JSON.parse(StringIO.new(diskUsedPercentageRecord))) end rescue => errorStr @log.info "Error in getDiskUsageMetricRecords: #{errorStr}" @@ -469,7 +468,7 @@ def getMetricRecords(record) dimValues: dimValues, metricValue: v, } - records.push(Yajl::Parser.parse(StringIO.new(metricRecord))) + records.push(JSON.parse(StringIO.new(metricRecord))) #@log.info "pushed mdmgenericmetric: #{k},#{v}" end } @@ -545,7 +544,7 @@ def getNodeResourceMetricRecords(record, metric_name, metric_value, percentage_m metricmaxvalue: metric_value, metricsumvalue: metric_value, } - records.push(Yajl::Parser.parse(StringIO.new(custommetricrecord))) + records.push(JSON.parse(StringIO.new(custommetricrecord))) if !percentage_metric_value.nil? additional_record = MdmAlertTemplates::Node_resource_metrics_template % { @@ -558,7 +557,7 @@ def getNodeResourceMetricRecords(record, metric_name, metric_value, percentage_m metricmaxvalue: percentage_metric_value, metricsumvalue: percentage_metric_value, } - records.push(Yajl::Parser.parse(StringIO.new(additional_record))) + records.push(JSON.parse(StringIO.new(additional_record))) end if !allocatable_percentage_metric_value.nil? @@ -572,7 +571,7 @@ def getNodeResourceMetricRecords(record, metric_name, metric_value, percentage_m metricmaxvalue: allocatable_percentage_metric_value, metricsumvalue: allocatable_percentage_metric_value, } - records.push(Yajl::Parser.parse(StringIO.new(additional_record))) + records.push(JSON.parse(StringIO.new(additional_record))) end rescue => errorStr @log.info "Error in getNodeResourceMetricRecords: #{errorStr}" diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 39b8c1c96..43707b91f 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -3,7 +3,7 @@ require "net/http" require "net/https" require "uri" -require "yajl/json_gem" +require "json" require "base64" require "time" require_relative "KubernetesApiClient" diff --git a/source/plugins/ruby/filter_cadvisor2mdm.rb b/source/plugins/ruby/filter_cadvisor2mdm.rb index 6bafa372a..621c94992 100644 --- a/source/plugins/ruby/filter_cadvisor2mdm.rb +++ b/source/plugins/ruby/filter_cadvisor2mdm.rb @@ -6,7 +6,7 @@ module Fluent::Plugin require "logger" - require "yajl/json_gem" + require "json" require_relative "oms_common" require_relative "CustomMetricsUtils" require_relative "kubelet_utils" diff --git a/source/plugins/ruby/filter_inventory2mdm.rb b/source/plugins/ruby/filter_inventory2mdm.rb index 509ac608e..165bb63cf 100644 --- a/source/plugins/ruby/filter_inventory2mdm.rb +++ b/source/plugins/ruby/filter_inventory2mdm.rb @@ -6,7 +6,7 @@ module Fluent::Plugin require 'logger' - require 'yajl/json_gem' + require 'json' require_relative 'oms_common' require_relative 'CustomMetricsUtils' diff --git a/source/plugins/ruby/filter_telegraf2mdm.rb b/source/plugins/ruby/filter_telegraf2mdm.rb index fd71f1682..0819afdb7 100644 --- a/source/plugins/ruby/filter_telegraf2mdm.rb +++ b/source/plugins/ruby/filter_telegraf2mdm.rb @@ -6,7 +6,7 @@ module Fluent::Plugin require "logger" - require "yajl/json_gem" + require "json" require_relative "oms_common" require_relative "kubelet_utils" require_relative "MdmMetricsGenerator" diff --git a/source/plugins/ruby/in_cadvisor_perf.rb b/source/plugins/ruby/in_cadvisor_perf.rb index 901ecefab..d929e86fb 100644 --- a/source/plugins/ruby/in_cadvisor_perf.rb +++ b/source/plugins/ruby/in_cadvisor_perf.rb @@ -14,7 +14,7 @@ class CAdvisor_Perf_Input < Input def initialize super require "yaml" - require "yajl/json_gem" + require "json" require "time" require_relative "CAdvisorMetricsAPIClient" diff --git a/source/plugins/ruby/in_containerinventory.rb b/source/plugins/ruby/in_containerinventory.rb index c8ffe7d05..aeb70c68a 100644 --- a/source/plugins/ruby/in_containerinventory.rb +++ b/source/plugins/ruby/in_containerinventory.rb @@ -11,7 +11,7 @@ class Container_Inventory_Input < Input def initialize super - require "yajl/json_gem" + require "json" require "time" require_relative "ContainerInventoryState" require_relative "ApplicationInsightsUtility" diff --git a/source/plugins/ruby/in_kube_events.rb b/source/plugins/ruby/in_kube_events.rb index deeae6e14..6ccb02c54 100644 --- a/source/plugins/ruby/in_kube_events.rb +++ b/source/plugins/ruby/in_kube_events.rb @@ -10,8 +10,7 @@ class Kube_Event_Input < Input def initialize super - require "yajl/json_gem" - require "yajl" + require "json" require "time" require_relative "KubernetesApiClient" diff --git a/source/plugins/ruby/in_kube_nodes.rb b/source/plugins/ruby/in_kube_nodes.rb index 5a52a089b..f497f21b9 100644 --- a/source/plugins/ruby/in_kube_nodes.rb +++ b/source/plugins/ruby/in_kube_nodes.rb @@ -15,8 +15,7 @@ def initialize(kubernetesApiClient = nil, super() require "yaml" - require "yajl/json_gem" - require "yajl" + require "json" require "time" require_relative "KubernetesApiClient" diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index f979ef7c5..fb752acd9 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -15,8 +15,7 @@ class Kube_PodInventory_Input < Input def initialize super require "yaml" - require "yajl/json_gem" - require "yajl" + require "json" require "set" require "time" @@ -154,7 +153,7 @@ def enumerate(podList = nil) if !serviceInfo.nil? $log.info("in_kube_podinventory::enumerate:Start:Parsing services data using yajl @ #{Time.now.utc.iso8601}") - serviceList = Yajl::Parser.parse(StringIO.new(serviceInfo.body)) + serviceList = JSON.parse(StringIO.new(serviceInfo.body)) $log.info("in_kube_podinventory::enumerate:End:Parsing services data using yajl @ #{Time.now.utc.iso8601}") serviceInfo = nil # service inventory records much smaller and fixed size compared to serviceList diff --git a/source/plugins/ruby/in_kube_pvinventory.rb b/source/plugins/ruby/in_kube_pvinventory.rb index fccfd459d..1e25e4057 100644 --- a/source/plugins/ruby/in_kube_pvinventory.rb +++ b/source/plugins/ruby/in_kube_pvinventory.rb @@ -12,8 +12,7 @@ class Kube_PVInventory_Input < Input def initialize super require "yaml" - require "yajl/json_gem" - require "yajl" + require "json" require "time" require_relative "KubernetesApiClient" require_relative "ApplicationInsightsUtility" diff --git a/source/plugins/ruby/in_kubestate_deployments.rb b/source/plugins/ruby/in_kubestate_deployments.rb index 0b563a890..92e6318b9 100644 --- a/source/plugins/ruby/in_kubestate_deployments.rb +++ b/source/plugins/ruby/in_kubestate_deployments.rb @@ -12,8 +12,7 @@ class Kube_Kubestate_Deployments_Input < Input def initialize super - require "yajl/json_gem" - require "yajl" + require "json" require "date" require "time" diff --git a/source/plugins/ruby/in_kubestate_hpa.rb b/source/plugins/ruby/in_kubestate_hpa.rb index 178f7944f..7f7e3aac5 100644 --- a/source/plugins/ruby/in_kubestate_hpa.rb +++ b/source/plugins/ruby/in_kubestate_hpa.rb @@ -10,8 +10,7 @@ class Kube_Kubestate_HPA_Input < Input def initialize super - require "yajl/json_gem" - require "yajl" + require "json" require "time" require_relative "KubernetesApiClient" diff --git a/source/plugins/ruby/in_win_cadvisor_perf.rb b/source/plugins/ruby/in_win_cadvisor_perf.rb index dd462fdf2..841c4867a 100644 --- a/source/plugins/ruby/in_win_cadvisor_perf.rb +++ b/source/plugins/ruby/in_win_cadvisor_perf.rb @@ -12,7 +12,7 @@ class Win_CAdvisor_Perf_Input < Input def initialize super require "yaml" - require "yajl/json_gem" + require "json" require "time" require_relative "CAdvisorMetricsAPIClient" diff --git a/source/plugins/ruby/kubelet_utils.rb b/source/plugins/ruby/kubelet_utils.rb index 368ca8639..f419986ad 100644 --- a/source/plugins/ruby/kubelet_utils.rb +++ b/source/plugins/ruby/kubelet_utils.rb @@ -3,7 +3,7 @@ # frozen_string_literal: true require "logger" -require "yajl/json_gem" +require "json" require_relative "CAdvisorMetricsAPIClient" require_relative "KubernetesApiClient" require "bigdecimal" @@ -165,7 +165,7 @@ def get_all_container_limits containerResourceDimensionHash = {} response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) if !response.nil? && !response.body.nil? && !response.body.empty? - podInventory = Yajl::Parser.parse(StringIO.new(response.body)) + podInventory = JSON.parse(StringIO.new(response.body)) podInventory["items"].each do |items| @log.info "in pod inventory items..." podNameSpace = items["metadata"]["namespace"] diff --git a/source/plugins/ruby/kubernetes_container_inventory.rb b/source/plugins/ruby/kubernetes_container_inventory.rb index 82e36c8cc..1f5129fe4 100644 --- a/source/plugins/ruby/kubernetes_container_inventory.rb +++ b/source/plugins/ruby/kubernetes_container_inventory.rb @@ -2,9 +2,8 @@ # frozen_string_literal: true class KubernetesContainerInventory - require "yajl/json_gem" - require "time" require "json" + require "time" require_relative "omslog" require_relative "ApplicationInsightsUtility" diff --git a/source/plugins/ruby/lib/application_insights/channel/contracts/json_serializable.rb b/source/plugins/ruby/lib/application_insights/channel/contracts/json_serializable.rb index 60838e215..8f4677044 100644 --- a/source/plugins/ruby/lib/application_insights/channel/contracts/json_serializable.rb +++ b/source/plugins/ruby/lib/application_insights/channel/contracts/json_serializable.rb @@ -1,4 +1,4 @@ -require 'yajl/json_gem' +require 'json' module ApplicationInsights module Channel diff --git a/source/plugins/ruby/lib/application_insights/channel/sender_base.rb b/source/plugins/ruby/lib/application_insights/channel/sender_base.rb index e5a4dea62..f5102c27a 100644 --- a/source/plugins/ruby/lib/application_insights/channel/sender_base.rb +++ b/source/plugins/ruby/lib/application_insights/channel/sender_base.rb @@ -1,4 +1,4 @@ -require "yajl/json_gem" +require "json" require "net/http" require "openssl" require "stringio" diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index c83972f11..a77a63634 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -13,7 +13,7 @@ def initialize require "net/http" require "net/https" require "uri" - require "yajl/json_gem" + require "json" require_relative "KubernetesApiClient" require_relative "ApplicationInsightsUtility" require_relative "constants" diff --git a/source/plugins/ruby/podinventory_to_mdm.rb b/source/plugins/ruby/podinventory_to_mdm.rb index c24a91a87..5b8fc00ff 100644 --- a/source/plugins/ruby/podinventory_to_mdm.rb +++ b/source/plugins/ruby/podinventory_to_mdm.rb @@ -3,7 +3,7 @@ # frozen_string_literal: true require "logger" -require "yajl/json_gem" +require "json" require "time" require_relative "oms_common" require_relative "CustomMetricsUtils" From 67db7139a2278c04ad0dca080cb7af348cbd0690 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Wed, 18 May 2022 21:36:05 -0700 Subject: [PATCH 20/36] remove stringio --- source/plugins/ruby/KubernetesApiClient.rb | 34 ++++++++++----------- source/plugins/ruby/MdmMetricsGenerator.rb | 22 ++++++------- source/plugins/ruby/in_kube_podinventory.rb | 4 +-- source/plugins/ruby/kubelet_utils.rb | 32 +++++++++---------- 4 files changed, 46 insertions(+), 46 deletions(-) diff --git a/source/plugins/ruby/KubernetesApiClient.rb b/source/plugins/ruby/KubernetesApiClient.rb index 392f67263..d7eaca7e7 100644 --- a/source/plugins/ruby/KubernetesApiClient.rb +++ b/source/plugins/ruby/KubernetesApiClient.rb @@ -88,7 +88,7 @@ def getTokenStr end end - def getClusterRegion(env=ENV) + def getClusterRegion(env = ENV) if env["AKS_REGION"] return env["AKS_REGION"] else @@ -97,7 +97,7 @@ def getClusterRegion(env=ENV) end end - def getResourceUri(resource, api_group, env=ENV) + def getResourceUri(resource, api_group, env = ENV) begin if env["KUBERNETES_SERVICE_HOST"] && env["KUBERNETES_PORT_443_TCP_PORT"] if api_group.nil? @@ -114,7 +114,7 @@ def getResourceUri(resource, api_group, env=ENV) end end - def getClusterName(env=ENV) + def getClusterName(env = ENV) return @@ClusterName if !@@ClusterName.nil? @@ClusterName = "None" begin @@ -148,7 +148,7 @@ def getClusterName(env=ENV) return @@ClusterName end - def getClusterId(env=ENV) + def getClusterId(env = ENV) return @@ClusterId if !@@ClusterId.nil? #By default initialize ClusterId to ClusterName. # In ACS/On-prem, we need to figure out how we can generate ClusterId @@ -456,19 +456,19 @@ def getContainerResourceRequestsAndLimits(pod, metricCategory, metricNameToColle metricCollection = {} metricCollection["CounterName"] = metricNametoReturn metricCollection["Value"] = metricValue - + metricProps["json_Collections"] = [] - metricCollections = [] - metricCollections.push(metricCollection) + metricCollections = [] + metricCollections.push(metricCollection) metricProps["json_Collections"] = metricCollections.to_json - metricItems.push(metricProps) + metricItems.push(metricProps) #No container level limit for the given metric, so default to node level limit else nodeMetricsHashKey = clusterId + "/" + nodeName + "_" + "allocatable" + "_" + metricNameToCollect if (metricCategory == "limits" && @@NodeMetrics.has_key?(nodeMetricsHashKey)) metricValue = @@NodeMetrics[nodeMetricsHashKey] #@Log.info("Limits not set for container #{clusterId + "/" + podUid + "/" + containerName} using node level limits: #{nodeMetricsHashKey}=#{metricValue} ") - + metricProps = {} metricProps["Timestamp"] = metricTime metricProps["Host"] = nodeName @@ -481,10 +481,10 @@ def getContainerResourceRequestsAndLimits(pod, metricCategory, metricNameToColle metricCollection["CounterName"] = metricNametoReturn metricCollection["Value"] = metricValue metricProps["json_Collections"] = [] - metricCollections = [] - metricCollections.push(metricCollection) + metricCollections = [] + metricCollections.push(metricCollection) metricProps["json_Collections"] = metricCollections.to_json - metricItems.push(metricProps) + metricItems.push(metricProps) end end end @@ -615,11 +615,11 @@ def parseNodeLimitsFromNodeItem(node, metricCategory, metricNameToCollect, metri metricCollection["CounterName"] = metricNametoReturn metricCollection["Value"] = metricValue metricCollections = [] - metricCollections.push(metricCollection) - + metricCollections.push(metricCollection) + metricItem["json_Collections"] = [] metricItem["json_Collections"] = metricCollections.to_json - + #push node level metrics to a inmem hash so that we can use it looking up at container level. #Currently if container level cpu & memory limits are not defined we default to node level limits @@NodeMetrics[clusterId + "/" + node["metadata"]["name"] + "_" + metricCategory + "_" + metricNameToCollect] = metricValue @@ -763,7 +763,7 @@ def getResourcesAndContinuationToken(uri, api_group: nil) @Log.info "KubernetesApiClient::getResourcesAndContinuationToken : Done getting resources from Kube API using url: #{uri} @ #{Time.now.utc.iso8601}" if !resourceInfo.nil? @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:Start:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" - resourceInventory = JSON.parse(StringIO.new(resourceInfo.body)) + resourceInventory = JSON.parse(resourceInfo.body) @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:End:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" resourceInfo = nil end @@ -778,7 +778,7 @@ def getResourcesAndContinuationToken(uri, api_group: nil) return continuationToken, resourceInventory end #getResourcesAndContinuationToken - def getKubeAPIServerUrl(env=ENV) + def getKubeAPIServerUrl(env = ENV) apiServerUrl = nil begin if env["KUBERNETES_SERVICE_HOST"] && env["KUBERNETES_PORT_443_TCP_PORT"] diff --git a/source/plugins/ruby/MdmMetricsGenerator.rb b/source/plugins/ruby/MdmMetricsGenerator.rb index cfd2404db..8ea06fe71 100644 --- a/source/plugins/ruby/MdmMetricsGenerator.rb +++ b/source/plugins/ruby/MdmMetricsGenerator.rb @@ -3,7 +3,7 @@ class MdmMetricsGenerator require "logger" - require "json" + require "json" require_relative "MdmAlertTemplates" require_relative "ApplicationInsightsUtility" require_relative "constants" @@ -139,7 +139,7 @@ def appendPodMetrics(records, metricName, metricHash, batch_time, metricsTemplat containerCountMetricValue: value, } end - records.push(JSON.parse(StringIO.new(record))) + records.push(JSON.parse(record)) } else @log.info "No records found in hash for metric: #{metricName}" @@ -333,7 +333,7 @@ def getContainerResourceUtilMetricRecords(recordTimeStamp, metricName, percentag containerResourceUtilizationPercentage: percentageMetricValue, thresholdPercentageDimValue: thresholdPercentage, } - records.push(JSON.parse(StringIO.new(resourceUtilRecord))) + records.push(JSON.parse(resourceUtilRecord)) # Adding another metric for threshold violation resourceThresholdViolatedRecord = MdmAlertTemplates::Container_resource_threshold_violation_template % { @@ -346,7 +346,7 @@ def getContainerResourceUtilMetricRecords(recordTimeStamp, metricName, percentag containerResourceThresholdViolated: isZeroFill ? 0 : 1, thresholdPercentageDimValue: thresholdPercentage, } - records.push(JSON.parse(StringIO.new(resourceThresholdViolatedRecord))) + records.push(JSON.parse(resourceThresholdViolatedRecord)) rescue => errorStr @log.info "Error in getContainerResourceUtilMetricRecords: #{errorStr}" ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) @@ -373,7 +373,7 @@ def getPVResourceUtilMetricRecords(recordTimeStamp, metricName, computer, percen pvResourceUtilizationPercentage: percentageMetricValue, thresholdPercentageDimValue: thresholdPercentage, } - records.push(JSON.parse(StringIO.new(resourceUtilRecord))) + records.push(JSON.parse(resourceUtilRecord)) # Adding another metric for threshold violation resourceThresholdViolatedRecord = MdmAlertTemplates::PV_resource_threshold_violation_template % { @@ -386,7 +386,7 @@ def getPVResourceUtilMetricRecords(recordTimeStamp, metricName, computer, percen pvResourceThresholdViolated: isZeroFill ? 0 : 1, thresholdPercentageDimValue: thresholdPercentage, } - records.push(JSON.parse(StringIO.new(resourceThresholdViolatedRecord))) + records.push(JSON.parse(resourceThresholdViolatedRecord)) rescue => errorStr @log.info "Error in getPVResourceUtilMetricRecords: #{errorStr}" ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) @@ -417,7 +417,7 @@ def getDiskUsageMetricRecords(record) devicevalue: deviceName, diskUsagePercentageValue: usedPercent, } - records.push(JSON.parse(StringIO.new(diskUsedPercentageRecord))) + records.push(JSON.parse(diskUsedPercentageRecord)) end rescue => errorStr @log.info "Error in getDiskUsageMetricRecords: #{errorStr}" @@ -468,7 +468,7 @@ def getMetricRecords(record) dimValues: dimValues, metricValue: v, } - records.push(JSON.parse(StringIO.new(metricRecord))) + records.push(JSON.parse(metricRecord)) #@log.info "pushed mdmgenericmetric: #{k},#{v}" end } @@ -544,7 +544,7 @@ def getNodeResourceMetricRecords(record, metric_name, metric_value, percentage_m metricmaxvalue: metric_value, metricsumvalue: metric_value, } - records.push(JSON.parse(StringIO.new(custommetricrecord))) + records.push(JSON.parse(custommetricrecord)) if !percentage_metric_value.nil? additional_record = MdmAlertTemplates::Node_resource_metrics_template % { @@ -557,7 +557,7 @@ def getNodeResourceMetricRecords(record, metric_name, metric_value, percentage_m metricmaxvalue: percentage_metric_value, metricsumvalue: percentage_metric_value, } - records.push(JSON.parse(StringIO.new(additional_record))) + records.push(JSON.parse(additional_record)) end if !allocatable_percentage_metric_value.nil? @@ -571,7 +571,7 @@ def getNodeResourceMetricRecords(record, metric_name, metric_value, percentage_m metricmaxvalue: allocatable_percentage_metric_value, metricsumvalue: allocatable_percentage_metric_value, } - records.push(JSON.parse(StringIO.new(additional_record))) + records.push(JSON.parse(additional_record)) end rescue => errorStr @log.info "Error in getNodeResourceMetricRecords: #{errorStr}" diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index fb752acd9..83b1bb2d7 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -15,7 +15,7 @@ class Kube_PodInventory_Input < Input def initialize super require "yaml" - require "json" + require "json" require "set" require "time" @@ -153,7 +153,7 @@ def enumerate(podList = nil) if !serviceInfo.nil? $log.info("in_kube_podinventory::enumerate:Start:Parsing services data using yajl @ #{Time.now.utc.iso8601}") - serviceList = JSON.parse(StringIO.new(serviceInfo.body)) + serviceList = JSON.parse(serviceInfo.body) $log.info("in_kube_podinventory::enumerate:End:Parsing services data using yajl @ #{Time.now.utc.iso8601}") serviceInfo = nil # service inventory records much smaller and fixed size compared to serviceList diff --git a/source/plugins/ruby/kubelet_utils.rb b/source/plugins/ruby/kubelet_utils.rb index f419986ad..b986f2ab7 100644 --- a/source/plugins/ruby/kubelet_utils.rb +++ b/source/plugins/ruby/kubelet_utils.rb @@ -52,7 +52,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) cpu_allocatable = 1.0 memory_allocatable = 1.0 - + allocatable_response = CAdvisorMetricsAPIClient.getCongifzCAdvisor(winNode: nil) parsed_response = JSON.parse(allocatable_response.body) @@ -66,7 +66,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::kubereserved_cpu: #{errorStr}" kubereserved_cpu = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::kubereserved_cpu: #{errorStr}") - end + end begin kubereserved_memory = parsed_response["kubeletconfig"]["kubeReserved"]["memory"] @@ -78,7 +78,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::kubereserved_memory: #{errorStr}" kubereserved_memory = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::kubereserved_memory: #{errorStr}") - end + end begin systemReserved_cpu = parsed_response["kubeletconfig"]["systemReserved"]["cpu"] if systemReserved_cpu.nil? || systemReserved_cpu == "" @@ -90,7 +90,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::systemReserved_cpu: #{errorStr}" systemReserved_cpu = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::systemReserved_cpu: #{errorStr}") - end + end begin explicitlyReserved_cpu = parsed_response["kubeletconfig"]["reservedCPUs"] @@ -103,19 +103,19 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::explicitlyReserved_cpu: #{errorStr}" explicitlyReserved_cpu = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::explicitlyReserved_cpu: #{errorStr}") - end + end begin - systemReserved_memory = parsed_response["kubeletconfig"]["systemReserved"]["memory"] - if systemReserved_memory.nil? || systemReserved_memory == "" + systemReserved_memory = parsed_response["kubeletconfig"]["systemReserved"]["memory"] + if systemReserved_memory.nil? || systemReserved_memory == "" systemReserved_memory = "0.0" - end - @log.info "get_node_allocatable::systemReserved_memory #{systemReserved_memory}" + end + @log.info "get_node_allocatable::systemReserved_memory #{systemReserved_memory}" rescue => errorStr - @log.error "Error in get_node_allocatable::systemReserved_memory: #{errorStr}" - systemReserved_memory = "0.0" + @log.error "Error in get_node_allocatable::systemReserved_memory: #{errorStr}" + systemReserved_memory = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::systemReserved_memory: #{errorStr}") - end + end begin evictionHard_memory = parsed_response["kubeletconfig"]["evictionHard"]["memory.available"] @@ -127,16 +127,16 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::evictionHard_memory: #{errorStr}" evictionHard_memory = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::evictionHard_memory: #{errorStr}") - end + end # do calculation in nanocore since that's what KubernetesApiClient.getMetricNumericValue expects cpu_capacity_number = cpu_capacity.to_i * 1000.0 ** 2 # subtract to get allocatable. Formula : Allocatable = Capacity - ( kube reserved + system reserved + eviction threshold ) # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable if KubernetesApiClient.getMetricNumericValue("cpu", explicitlyReserved_cpu) > 0 - cpu_allocatable = cpu_capacity_number - KubernetesApiClient.getMetricNumericValue("cpu", explicitlyReserved_cpu) + cpu_allocatable = cpu_capacity_number - KubernetesApiClient.getMetricNumericValue("cpu", explicitlyReserved_cpu) else - cpu_allocatable = cpu_capacity_number - (KubernetesApiClient.getMetricNumericValue("cpu", kubereserved_cpu) + KubernetesApiClient.getMetricNumericValue("cpu", systemReserved_cpu)) + cpu_allocatable = cpu_capacity_number - (KubernetesApiClient.getMetricNumericValue("cpu", kubereserved_cpu) + KubernetesApiClient.getMetricNumericValue("cpu", systemReserved_cpu)) end # convert back to units similar to what we get for capacity cpu_allocatable = cpu_allocatable / (1000.0 ** 2) @@ -165,7 +165,7 @@ def get_all_container_limits containerResourceDimensionHash = {} response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) if !response.nil? && !response.body.nil? && !response.body.empty? - podInventory = JSON.parse(StringIO.new(response.body)) + podInventory = JSON.parse(response.body) podInventory["items"].each do |items| @log.info "in pod inventory items..." podNameSpace = items["metadata"]["namespace"] From 13a0595011c692adc1b848f9fdee7dccfb4c1e90 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Fri, 27 May 2022 19:48:10 +0000 Subject: [PATCH 21/36] use rvm ruby 2.7.5 --- kubernetes/linux/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index f43fc602a..768d9230d 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -24,8 +24,8 @@ if [ -f /etc/profile.d/rvm.sh ]; then echo "[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh" >> ~/.bashrc fi -rvm install 3.1.0 -rvm --default use 3.1.0 +rvm install 2.7.5 +rvm --default use 2.7.5 sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ From f00fc7b9c78fb9ba7909040ee58f8cbf8ac24684 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Fri, 27 May 2022 20:11:10 +0000 Subject: [PATCH 22/36] use old trivyignore --- .trivyignore | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.trivyignore b/.trivyignore index d8be7f39e..e10a2afa5 100644 --- a/.trivyignore +++ b/.trivyignore @@ -5,4 +5,14 @@ CVE-2019-3826 CVE-2022-27191 #still present in mdsd telegraf -CVE-2021-42836 \ No newline at end of file +CVE-2021-42836 + +# ruby in /usr/lib +CVE-2020-36327 +CVE-2021-43809 +CVE-2021-41816 +CVE-2021-41819 +CVE-2021-31799 +CVE-2021-28965 + +CVE-2022-1664 \ No newline at end of file From 14216bee0822f748c73326f6d9770e573462480a Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 31 May 2022 22:28:40 +0000 Subject: [PATCH 23/36] try with 3.0.2 --- kubernetes/linux/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 768d9230d..7f1afd3ce 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -24,8 +24,8 @@ if [ -f /etc/profile.d/rvm.sh ]; then echo "[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh" >> ~/.bashrc fi -rvm install 2.7.5 -rvm --default use 2.7.5 +rvm install 3.0.2 +rvm --default use 3.0.2 sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ From 5fcf4d83fefc0e553f95fb9f894c1a5ee5df930f Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 31 May 2022 22:44:57 +0000 Subject: [PATCH 24/36] add vuln --- .trivyignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.trivyignore b/.trivyignore index e10a2afa5..020dc146f 100644 --- a/.trivyignore +++ b/.trivyignore @@ -14,5 +14,6 @@ CVE-2021-41816 CVE-2021-41819 CVE-2021-31799 CVE-2021-28965 +CVE-2021-41817 CVE-2022-1664 \ No newline at end of file From 0b8761b631a444b63dcde9c046b3488cdae45641 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Fri, 8 Jul 2022 19:18:07 +0000 Subject: [PATCH 25/36] use default json parser --- source/plugins/ruby/KubernetesApiClient.rb | 2 +- source/plugins/ruby/WatchStream.rb | 4 ++-- source/plugins/ruby/in_kube_perfinventory.rb | 5 ++--- source/plugins/ruby/in_kube_podinventory.rb | 2 +- source/plugins/ruby/in_kube_podmdminventory.rb | 5 ++--- source/plugins/ruby/podinventory_to_mdm.rb | 2 +- 6 files changed, 9 insertions(+), 11 deletions(-) diff --git a/source/plugins/ruby/KubernetesApiClient.rb b/source/plugins/ruby/KubernetesApiClient.rb index 907c437fb..7fa16e459 100644 --- a/source/plugins/ruby/KubernetesApiClient.rb +++ b/source/plugins/ruby/KubernetesApiClient.rb @@ -802,7 +802,7 @@ def getResourcesAndContinuationTokenV2(uri, api_group: nil) @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2 : Done getting resources from Kube API using url: #{uri} @ #{Time.now.utc.iso8601}" if !responseCode.nil? && responseCode == "200" && !resourceInfo.nil? @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2:Start:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" - resourceInventory = Yajl::Parser.parse(StringIO.new(resourceInfo.body)) + resourceInventory = JSON.parse(resourceInfo.body) @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2:End:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" resourceInfo = nil end diff --git a/source/plugins/ruby/WatchStream.rb b/source/plugins/ruby/WatchStream.rb index 6cc850450..78ce25dd5 100644 --- a/source/plugins/ruby/WatchStream.rb +++ b/source/plugins/ruby/WatchStream.rb @@ -3,7 +3,7 @@ require "net/http" require "net/https" -require "yajl/json_gem" +require "json" require "logger" require "time" @@ -50,7 +50,7 @@ def each response.read_body do |chunk| buffer << chunk while (line = buffer.slice!(/.+\n/)) - yield(Yajl::Parser.parse(StringIO.new(line.chomp))) + yield(JSON.parse(line.chomp)) end end end diff --git a/source/plugins/ruby/in_kube_perfinventory.rb b/source/plugins/ruby/in_kube_perfinventory.rb index 20589167b..d083133e4 100644 --- a/source/plugins/ruby/in_kube_perfinventory.rb +++ b/source/plugins/ruby/in_kube_perfinventory.rb @@ -10,8 +10,7 @@ class Kube_PerfInventory_Input < Input def initialize super require "yaml" - require "yajl/json_gem" - require "yajl" + require "json" require "set" require "time" require "net/http" @@ -407,7 +406,7 @@ def getNodeAllocatableRecords() isAcquiredLock = f.flock(File::LOCK_EX | File::LOCK_NB) raise "in_kube_perfinventory:getNodeAllocatableRecords:Failed to acquire file lock @ #{Time.now.utc.iso8601}" if !isAcquiredLock startTime = (Time.now.to_f * 1000).to_i - nodeAllocatableRecords = Yajl::Parser.parse(f) + nodeAllocatableRecords = JSON.parse(f) timetakenMs = ((Time.now.to_f * 1000).to_i - startTime) $log.info "in_kube_perfinventory:getNodeAllocatableRecords:Number of Node Allocatable records: #{nodeAllocatableRecords.length} with time taken(ms) for read: #{timetakenMs} @ #{Time.now.utc.iso8601}" else diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index b509c3ae3..e9c512cfd 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -916,7 +916,7 @@ def watch_services $log.info("in_kube_podinventory::watch_services: Done getting services from Kube API @ #{Time.now.utc.iso8601}") if !serviceInfo.nil? $log.info("in_kube_podinventory::watch_services:Start:Parsing services data using yajl @ #{Time.now.utc.iso8601}") - serviceInventory = Yajl::Parser.parse(StringIO.new(serviceInfo.body)) + serviceInventory = JSON.parse(serviceInfo.body) $log.info("in_kube_podinventory::watch_services:End:Parsing services data using yajl @ #{Time.now.utc.iso8601}") serviceInfo = nil if (!serviceInventory.nil? && !serviceInventory.empty?) diff --git a/source/plugins/ruby/in_kube_podmdminventory.rb b/source/plugins/ruby/in_kube_podmdminventory.rb index b872650d2..4bef0b3f6 100644 --- a/source/plugins/ruby/in_kube_podmdminventory.rb +++ b/source/plugins/ruby/in_kube_podmdminventory.rb @@ -14,8 +14,7 @@ class Kube_PodMDMInventory_Input < Input def initialize super require "yaml" - require "yajl/json_gem" - require "yajl" + require "json" require "set" require "time" require "net/http" @@ -187,7 +186,7 @@ def getMDMRecords() isAcquiredLock = f.flock(File::LOCK_EX | File::LOCK_NB) raise "in_kube_podmdminventory:getMDMRecords:Failed to acquire file lock @ #{Time.now.utc.iso8601}" if !isAcquiredLock startTime = (Time.now.to_f * 1000).to_i - mdmRecords = Yajl::Parser.parse(f) + mdmRecords = JSON.parse(f) timetakenMs = ((Time.now.to_f * 1000).to_i - startTime) if mdmRecords.nil? || mdmRecords.empty? || mdmRecords["items"].nil? || mdmRecords["collectionTime"] == @prevCollectionTime raise "in_kube_podmdminventory:getMDMRecords: either read mdmRecords is nil or empty or stale @ #{Time.now.utc.iso8601}" diff --git a/source/plugins/ruby/podinventory_to_mdm.rb b/source/plugins/ruby/podinventory_to_mdm.rb index 0f145ce41..5102274ed 100644 --- a/source/plugins/ruby/podinventory_to_mdm.rb +++ b/source/plugins/ruby/podinventory_to_mdm.rb @@ -129,7 +129,7 @@ def get_pod_inventory_mdm_records(batch_time) controllerNameDimValue: podControllerNameDimValue, podCountMetricValue: value, } - records.push(Yajl::Parser.parse(record)) + records.push(JSON.parse(record)) } #Add pod metric records From 99cbcec0a72e4a5e25a28cdae7e6bed706bdea74 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Fri, 8 Jul 2022 19:25:33 +0000 Subject: [PATCH 26/36] 3.0.4 default json --- .trivyignore | 5 +++++ kubernetes/linux/setup.sh | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.trivyignore b/.trivyignore index 46a9ebfe7..189c75162 100644 --- a/.trivyignore +++ b/.trivyignore @@ -17,3 +17,8 @@ CVE-2021-28965 #dpkg vulnerability in ubuntu CVE-2022-1304 + +# telegraf vulnerabilities +CVE-2022-29190 +CVE-2022-29222 +CVE-2022-29189 \ No newline at end of file diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 7f1afd3ce..9e0331071 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -24,8 +24,8 @@ if [ -f /etc/profile.d/rvm.sh ]; then echo "[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh" >> ~/.bashrc fi -rvm install 3.0.2 -rvm --default use 3.0.2 +rvm install 3.0.4 +rvm --default use 3.0.4 sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ From 9c4e9d60516cd818d3835a68267e7aa4efc7563a Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Fri, 8 Jul 2022 19:32:27 +0000 Subject: [PATCH 27/36] 3.1.1 default json --- kubernetes/linux/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 9e0331071..2a48d420d 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -24,8 +24,8 @@ if [ -f /etc/profile.d/rvm.sh ]; then echo "[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh" >> ~/.bashrc fi -rvm install 3.0.4 -rvm --default use 3.0.4 +rvm install 3.1.1 +rvm --default use 3.1.1 sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ From 356535585958ffc84c480b4d3937539fe4aacf38 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Fri, 8 Jul 2022 19:36:58 +0000 Subject: [PATCH 28/36] 2.7.5 default json --- kubernetes/linux/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 2a48d420d..768d9230d 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -24,8 +24,8 @@ if [ -f /etc/profile.d/rvm.sh ]; then echo "[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh" >> ~/.bashrc fi -rvm install 3.1.1 -rvm --default use 3.1.1 +rvm install 2.7.5 +rvm --default use 2.7.5 sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ From 7c64a23bd043809d0cc0b9d1cdfed49906fe7584 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 12 Jul 2022 01:02:05 +0000 Subject: [PATCH 29/36] fix bug + 2.7.5 --- source/plugins/ruby/in_kube_perfinventory.rb | 2 +- source/plugins/ruby/in_kube_podmdminventory.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/plugins/ruby/in_kube_perfinventory.rb b/source/plugins/ruby/in_kube_perfinventory.rb index d083133e4..25f9c93e8 100644 --- a/source/plugins/ruby/in_kube_perfinventory.rb +++ b/source/plugins/ruby/in_kube_perfinventory.rb @@ -406,7 +406,7 @@ def getNodeAllocatableRecords() isAcquiredLock = f.flock(File::LOCK_EX | File::LOCK_NB) raise "in_kube_perfinventory:getNodeAllocatableRecords:Failed to acquire file lock @ #{Time.now.utc.iso8601}" if !isAcquiredLock startTime = (Time.now.to_f * 1000).to_i - nodeAllocatableRecords = JSON.parse(f) + nodeAllocatableRecords = JSON.parse(f.read) timetakenMs = ((Time.now.to_f * 1000).to_i - startTime) $log.info "in_kube_perfinventory:getNodeAllocatableRecords:Number of Node Allocatable records: #{nodeAllocatableRecords.length} with time taken(ms) for read: #{timetakenMs} @ #{Time.now.utc.iso8601}" else diff --git a/source/plugins/ruby/in_kube_podmdminventory.rb b/source/plugins/ruby/in_kube_podmdminventory.rb index 4bef0b3f6..38e07d860 100644 --- a/source/plugins/ruby/in_kube_podmdminventory.rb +++ b/source/plugins/ruby/in_kube_podmdminventory.rb @@ -186,7 +186,7 @@ def getMDMRecords() isAcquiredLock = f.flock(File::LOCK_EX | File::LOCK_NB) raise "in_kube_podmdminventory:getMDMRecords:Failed to acquire file lock @ #{Time.now.utc.iso8601}" if !isAcquiredLock startTime = (Time.now.to_f * 1000).to_i - mdmRecords = JSON.parse(f) + mdmRecords = JSON.parse(f.read) timetakenMs = ((Time.now.to_f * 1000).to_i - startTime) if mdmRecords.nil? || mdmRecords.empty? || mdmRecords["items"].nil? || mdmRecords["collectionTime"] == @prevCollectionTime raise "in_kube_podmdminventory:getMDMRecords: either read mdmRecords is nil or empty or stale @ #{Time.now.utc.iso8601}" From 0f67aec6063a61c0b05af4590e3016da6fa54d51 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 12 Jul 2022 01:03:10 +0000 Subject: [PATCH 30/36] fix bug + 3.1.1 --- kubernetes/linux/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 768d9230d..2a48d420d 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -24,8 +24,8 @@ if [ -f /etc/profile.d/rvm.sh ]; then echo "[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh" >> ~/.bashrc fi -rvm install 2.7.5 -rvm --default use 2.7.5 +rvm install 3.1.1 +rvm --default use 3.1.1 sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ From 3a58122245bf9ca141055eefef38cf4158054dbe Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 12 Jul 2022 01:04:28 +0000 Subject: [PATCH 31/36] fix bug + 3.0.4 --- kubernetes/linux/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 2a48d420d..9e0331071 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -24,8 +24,8 @@ if [ -f /etc/profile.d/rvm.sh ]; then echo "[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh" >> ~/.bashrc fi -rvm install 3.1.1 -rvm --default use 3.1.1 +rvm install 3.0.4 +rvm --default use 3.0.4 sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ From e34c5af970d40855e713c575d1c04dd342c3b3de Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 12 Jul 2022 01:36:41 +0000 Subject: [PATCH 32/36] fix vuln + 3.0.4 --- .trivyignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.trivyignore b/.trivyignore index 189c75162..ca7946ef6 100644 --- a/.trivyignore +++ b/.trivyignore @@ -21,4 +21,5 @@ CVE-2022-1304 # telegraf vulnerabilities CVE-2022-29190 CVE-2022-29222 -CVE-2022-29189 \ No newline at end of file +CVE-2022-29189 +CVE-2022-31030 \ No newline at end of file From fdd1e1842c62cbd8134078d9434a18828ab6063a Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 12 Jul 2022 01:43:06 +0000 Subject: [PATCH 33/36] fix vuln + 3.1.1 --- kubernetes/linux/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 9e0331071..2a48d420d 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -24,8 +24,8 @@ if [ -f /etc/profile.d/rvm.sh ]; then echo "[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh" >> ~/.bashrc fi -rvm install 3.0.4 -rvm --default use 3.0.4 +rvm install 3.1.1 +rvm --default use 3.1.1 sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ From 233acfc373b8bb3ca3e1898b4d755d90f1be8862 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Tue, 12 Jul 2022 01:47:26 +0000 Subject: [PATCH 34/36] fix vuln + 2.7.5 --- kubernetes/linux/setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 2a48d420d..768d9230d 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -24,8 +24,8 @@ if [ -f /etc/profile.d/rvm.sh ]; then echo "[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh" >> ~/.bashrc fi -rvm install 3.1.1 -rvm --default use 3.1.1 +rvm install 2.7.5 +rvm --default use 2.7.5 sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ From bae092ad309bcff2f031b9bf2dd1ad1d50ca5fe1 Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Wed, 20 Jul 2022 05:23:06 +0000 Subject: [PATCH 35/36] use 3.1.1 --- .trivyignore | 14 ++------------ kubernetes/linux/setup.sh | 4 ++-- kubernetes/windows/Dockerfile | 2 +- kubernetes/windows/Dockerfile-dev-base-image | 2 +- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/.trivyignore b/.trivyignore index ca7946ef6..09381cd2e 100644 --- a/.trivyignore +++ b/.trivyignore @@ -4,17 +4,6 @@ CVE-2019-3826 CVE-2022-27191 -#still present in mdsd telegraf -CVE-2021-42836 - -# ruby in /usr/lib -CVE-2020-36327 -CVE-2021-43809 -CVE-2021-41816 -CVE-2021-41819 -CVE-2021-31799 -CVE-2021-28965 - #dpkg vulnerability in ubuntu CVE-2022-1304 @@ -22,4 +11,5 @@ CVE-2022-1304 CVE-2022-29190 CVE-2022-29222 CVE-2022-29189 -CVE-2022-31030 \ No newline at end of file +CVE-2022-31030 +CVE-2022-29526 diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 768d9230d..2a48d420d 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -24,8 +24,8 @@ if [ -f /etc/profile.d/rvm.sh ]; then echo "[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh" >> ~/.bashrc fi -rvm install 2.7.5 -rvm --default use 2.7.5 +rvm install 3.1.1 +rvm --default use 3.1.1 sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ diff --git a/kubernetes/windows/Dockerfile b/kubernetes/windows/Dockerfile index 1906a2439..0651dd227 100644 --- a/kubernetes/windows/Dockerfile +++ b/kubernetes/windows/Dockerfile @@ -11,7 +11,7 @@ ARG IMAGE_TAG=win-ciprod06142022 # Docker creates a layer for every RUN-Statement RUN powershell -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" # Fluentd depends on cool.io whose fat gem is only available for Ruby < 2.5, so need to specify --platform ruby when install Ruby > 2.5 and install msys2 to get dev tools -RUN choco install -y ruby --version 3.1.0.1 --params "'/InstallDir:C:\ruby31'" \ +RUN choco install -y ruby --version 3.1.1.1 --params "'/InstallDir:C:\ruby31'" \ && choco install -y msys2 --version 20211130.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby31\msys64'" \ && choco install -y vim diff --git a/kubernetes/windows/Dockerfile-dev-base-image b/kubernetes/windows/Dockerfile-dev-base-image index 89452256d..bcf28326e 100644 --- a/kubernetes/windows/Dockerfile-dev-base-image +++ b/kubernetes/windows/Dockerfile-dev-base-image @@ -8,7 +8,7 @@ LABEL vendor=Microsoft\ Corp \ # Docker creates a layer for every RUN-Statement RUN powershell -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" # Fluentd depends on cool.io whose fat gem is only available for Ruby < 2.5, so need to specify --platform ruby when install Ruby > 2.5 and install msys2 to get dev tools -RUN choco install -y ruby --version 3.1.0.1 --params "'/InstallDir:C:\ruby31'" \ +RUN choco install -y ruby --version 3.1.1.1 --params "'/InstallDir:C:\ruby31'" \ && choco install -y msys2 --version 20210604.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby31\msys64'" \ && choco install -y vim From d6bf72771ee6a3ac2fbf5f7a36534ffbabc3de7c Mon Sep 17 00:00:00 2001 From: Amol Agrawal Date: Fri, 22 Jul 2022 02:33:49 +0000 Subject: [PATCH 36/36] address PR comments --- kubernetes/windows/Dockerfile | 1 - source/plugins/ruby/KubernetesApiClient.rb | 8 ++++---- source/plugins/ruby/in_kube_podinventory.rb | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/kubernetes/windows/Dockerfile b/kubernetes/windows/Dockerfile index 0651dd227..019e9cda1 100644 --- a/kubernetes/windows/Dockerfile +++ b/kubernetes/windows/Dockerfile @@ -21,7 +21,6 @@ RUN refreshenv \ && echo gem: --no-document >> C:\ProgramData\gemrc \ && gem install cool.io -v 1.7.1 --platform ruby \ && gem install oj -v 3.3.10 \ -&& gem install json -v 2.2.0 \ && gem install fluentd -v 1.14.6 \ && gem install win32-service -v 1.0.1 \ && gem install win32-ipc -v 0.7.0 \ diff --git a/source/plugins/ruby/KubernetesApiClient.rb b/source/plugins/ruby/KubernetesApiClient.rb index 7fa16e459..612035625 100644 --- a/source/plugins/ruby/KubernetesApiClient.rb +++ b/source/plugins/ruby/KubernetesApiClient.rb @@ -801,9 +801,9 @@ def getResourcesAndContinuationTokenV2(uri, api_group: nil) responseCode, resourceInfo = getKubeResourceInfoV2(uri, api_group: api_group) @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2 : Done getting resources from Kube API using url: #{uri} @ #{Time.now.utc.iso8601}" if !responseCode.nil? && responseCode == "200" && !resourceInfo.nil? - @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2:Start:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" + @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2:Start:Parsing data for #{uri} using JSON @ #{Time.now.utc.iso8601}" resourceInventory = JSON.parse(resourceInfo.body) - @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2:End:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" + @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2:End:Parsing data for #{uri} using JSON @ #{Time.now.utc.iso8601}" resourceInfo = nil end if (!resourceInventory.nil? && !resourceInventory["metadata"].nil?) @@ -825,9 +825,9 @@ def getResourcesAndContinuationToken(uri, api_group: nil) resourceInfo = getKubeResourceInfo(uri, api_group: api_group) @Log.info "KubernetesApiClient::getResourcesAndContinuationToken : Done getting resources from Kube API using url: #{uri} @ #{Time.now.utc.iso8601}" if !resourceInfo.nil? - @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:Start:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" + @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:Start:Parsing data for #{uri} using JSON @ #{Time.now.utc.iso8601}" resourceInventory = JSON.parse(resourceInfo.body) - @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:End:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" + @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:End:Parsing data for #{uri} using JSON @ #{Time.now.utc.iso8601}" resourceInfo = nil end if (!resourceInventory.nil? && !resourceInventory["metadata"].nil?) diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index e9c512cfd..a1986bd4a 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -915,9 +915,9 @@ def watch_services else $log.info("in_kube_podinventory::watch_services: Done getting services from Kube API @ #{Time.now.utc.iso8601}") if !serviceInfo.nil? - $log.info("in_kube_podinventory::watch_services:Start:Parsing services data using yajl @ #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::watch_services:Start:Parsing services data using JSON @ #{Time.now.utc.iso8601}") serviceInventory = JSON.parse(serviceInfo.body) - $log.info("in_kube_podinventory::watch_services:End:Parsing services data using yajl @ #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::watch_services:End:Parsing services data using JSON @ #{Time.now.utc.iso8601}") serviceInfo = nil if (!serviceInventory.nil? && !serviceInventory.empty?) servicesResourceVersion = serviceInventory["metadata"]["resourceVersion"]