diff --git a/.trivyignore b/.trivyignore index 0e3293641..91ee2f5bb 100644 --- a/.trivyignore +++ b/.trivyignore @@ -1,19 +1,9 @@ -# related to telegraf -#[vishwa] - Fix telegraf & test all for next release - see work item #https://msazure.visualstudio.com/InfrastructureInsights/_workitems/edit/13322134 -# Unfixed as of 4/28/2022 +# telegraf vulnerabilities CVE-2019-3826 -CVE-2022-1996 CVE-2022-29190 CVE-2022-29222 CVE-2022-29189 - -# ruby in /usr/lib -CVE-2020-36327 -CVE-2021-43809 -CVE-2021-41816 -CVE-2021-41819 -CVE-2021-31799 -CVE-2021-28965 +CVE-2022-1996 #dpkg vulnerability in ubuntu CVE-2022-1304 diff --git a/README.md b/README.md index 60ed39901..2b0ab246e 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,6 @@ The general directory structure is: │ │ | ├── health/ - code for health feature │ │ | ├── lib/ - lib for app insights ruby and this code of application_insights gem │ │ | ... - plugins in, out and filters code in ruby -│ ├── toml-parser/ - code for parsing of toml configuration files ├── test/ - source code for tests │ ├── e2e/ - e2e tests to validate agent and e2e workflow(s) │ ├── unit-tests/ - unit tests code diff --git a/build/common/installer/scripts/tomlparser-agent-config.rb b/build/common/installer/scripts/tomlparser-agent-config.rb index ebe1e3982..a6b6ee0a1 100644 --- a/build/common/installer/scripts/tomlparser-agent-config.rb +++ b/build/common/installer/scripts/tomlparser-agent-config.rb @@ -2,11 +2,7 @@ #this should be require relative in Linux and require in windows, since it is a gem install on windows @os_type = ENV["OS_TYPE"] -if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 - require "tomlrb" -else - require_relative "tomlrb" -end +require "tomlrb" require_relative "ConfigParseErrorLogger" diff --git a/build/common/installer/scripts/tomlparser-mdm-metrics-config.rb b/build/common/installer/scripts/tomlparser-mdm-metrics-config.rb index b6a4419cf..17c1ca118 100644 --- a/build/common/installer/scripts/tomlparser-mdm-metrics-config.rb +++ b/build/common/installer/scripts/tomlparser-mdm-metrics-config.rb @@ -3,11 +3,7 @@ #this should be require relative in Linux and require in windows, since it is a gem install on windows @os_type = ENV["OS_TYPE"] -if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 - require "tomlrb" -else - require_relative "tomlrb" -end +require "tomlrb" require_relative "/etc/fluent/plugin/constants" require_relative "ConfigParseErrorLogger" diff --git a/build/common/installer/scripts/tomlparser-prom-agent-config.rb b/build/common/installer/scripts/tomlparser-prom-agent-config.rb index 664691a44..abc939f52 100644 --- a/build/common/installer/scripts/tomlparser-prom-agent-config.rb +++ b/build/common/installer/scripts/tomlparser-prom-agent-config.rb @@ -2,11 +2,7 @@ #this should be require relative in Linux and require in windows, since it is a gem install on windows @os_type = ENV["OS_TYPE"] -if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 - require "tomlrb" -else - require_relative "tomlrb" -end +require "tomlrb" require_relative "ConfigParseErrorLogger" diff --git a/build/common/installer/scripts/tomlparser-prom-customconfig.rb b/build/common/installer/scripts/tomlparser-prom-customconfig.rb index 642eadc14..76909b17c 100644 --- a/build/common/installer/scripts/tomlparser-prom-customconfig.rb +++ b/build/common/installer/scripts/tomlparser-prom-customconfig.rb @@ -2,12 +2,8 @@ #this should be require relative in Linux and require in windows, since it is a gem install on windows @os_type = ENV["OS_TYPE"] -if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 - require "tomlrb" -else - require_relative "tomlrb" -end -# require_relative "tomlrb" +require "tomlrb" + require_relative "ConfigParseErrorLogger" require "fileutils" diff --git a/build/common/installer/scripts/tomlparser.rb b/build/common/installer/scripts/tomlparser.rb index 6a2f3c6d6..6d3ee6e78 100644 --- a/build/common/installer/scripts/tomlparser.rb +++ b/build/common/installer/scripts/tomlparser.rb @@ -2,11 +2,7 @@ #this should be require relative in Linux and require in windows, since it is a gem install on windows @os_type = ENV["OS_TYPE"] -if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 - require "tomlrb" -else - require_relative "tomlrb" -end +require "tomlrb" require_relative "ConfigParseErrorLogger" diff --git a/build/linux/installer/datafiles/base_container.data b/build/linux/installer/datafiles/base_container.data index 92b494ae3..db7acb57b 100644 --- a/build/linux/installer/datafiles/base_container.data +++ b/build/linux/installer/datafiles/base_container.data @@ -20,15 +20,6 @@ MAINTAINER: 'Microsoft Corporation' /etc/opt/microsoft/docker-cimprov/container.conf; build/linux/installer/conf/container.conf; 644; root; root -/opt/tomlrb.rb; source/toml-parser/tomlrb.rb; 644; root; root -/opt/tomlrb/generated_parser.rb; source/toml-parser/tomlrb/generated_parser.rb; 644; root; root -/opt/tomlrb/handler.rb; source/toml-parser/tomlrb/handler.rb; 644; root; root -/opt/tomlrb/parser.rb; source/toml-parser/tomlrb/parser.rb; 644; root; root -/opt/tomlrb/parser.y; source/toml-parser/tomlrb/parser.y; 644; root; root -/opt/tomlrb/scanner.rb; source/toml-parser/tomlrb/scanner.rb; 644; root; root -/opt/tomlrb/string_utils.rb; source/toml-parser/tomlrb/string_utils.rb; 644; root; root -/opt/tomlrb/version.rb; source/toml-parser/tomlrb/version.rb; 644; root; root - /opt/td-agent-bit/bin/out_oms.so; intermediate/${{BUILD_CONFIGURATION}}/out_oms.so; 755; root; root /etc/opt/microsoft/docker-cimprov/prometheus-side-car.conf; build/linux/installer/conf/prometheus-side-car.conf; 644; root; root /etc/opt/microsoft/docker-cimprov/td-agent-bit.conf; build/linux/installer/conf/td-agent-bit.conf; 644; root; root diff --git a/build/linux/installer/scripts/tomlparser-metric-collection-config.rb b/build/linux/installer/scripts/tomlparser-metric-collection-config.rb index cee41312b..3001fdbaf 100644 --- a/build/linux/installer/scripts/tomlparser-metric-collection-config.rb +++ b/build/linux/installer/scripts/tomlparser-metric-collection-config.rb @@ -1,7 +1,7 @@ #!/usr/local/bin/ruby # frozen_string_literal: true -require_relative "tomlrb" +require "tomlrb" require_relative "ConfigParseErrorLogger" require_relative "/etc/fluent/plugin/constants" diff --git a/build/linux/installer/scripts/tomlparser-npm-config.rb b/build/linux/installer/scripts/tomlparser-npm-config.rb index e8cf216fd..664ed4f0b 100644 --- a/build/linux/installer/scripts/tomlparser-npm-config.rb +++ b/build/linux/installer/scripts/tomlparser-npm-config.rb @@ -2,11 +2,7 @@ #this should be require relative in Linux and require in windows, since it is a gem install on windows @os_type = ENV["OS_TYPE"] -if !@os_type.nil? && !@os_type.empty? && @os_type.strip.casecmp("windows") == 0 - require "tomlrb" -else - require_relative "tomlrb" -end +require "tomlrb" require_relative "ConfigParseErrorLogger" diff --git a/build/linux/installer/scripts/tomlparser-osm-config.rb b/build/linux/installer/scripts/tomlparser-osm-config.rb index 096064db8..2ac5ef387 100644 --- a/build/linux/installer/scripts/tomlparser-osm-config.rb +++ b/build/linux/installer/scripts/tomlparser-osm-config.rb @@ -1,6 +1,6 @@ #!/usr/local/bin/ruby -require_relative "tomlrb" +require "tomlrb" require "fileutils" require_relative "ConfigParseErrorLogger" diff --git a/build/windows/installer/conf/fluent.conf b/build/windows/installer/conf/fluent.conf index a78ac58fa..73d62a3ff 100644 --- a/build/windows/installer/conf/fluent.conf +++ b/build/windows/installer/conf/fluent.conf @@ -1,5 +1,5 @@ - type heartbeat_request + @type heartbeat_request run_interval 30m @log_level info diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index 1e00457d9..efb95698a 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -88,6 +88,8 @@ checkAgentOnboardingStatus() { fi } +# setup paths for ruby +[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh setReplicaSetSpecificConfig() { echo "num of fluentd workers:${NUM_OF_FLUENTD_WORKERS}" export FLUENTD_FLUSH_INTERVAL="20s" @@ -453,7 +455,7 @@ source ~/.bashrc if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then #Parse the configmap to set the right environment variables. - /usr/bin/ruby2.7 tomlparser.rb + ruby tomlparser.rb cat config_env_var | while read line; do echo $line >>~/.bashrc @@ -464,7 +466,7 @@ fi #Parse the configmap to set the right environment variables for agent config. #Note > tomlparser-agent-config.rb has to be parsed first before td-agent-bit-conf-customizer.rb for fbit agent settings if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then - /usr/bin/ruby2.7 tomlparser-agent-config.rb + ruby tomlparser-agent-config.rb cat agent_config_env_var | while read line; do echo $line >> ~/.bashrc @@ -472,7 +474,7 @@ if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then source agent_config_env_var #Parse the configmap to set the right environment variables for network policy manager (npm) integration. - /usr/bin/ruby2.7 tomlparser-npm-config.rb + ruby tomlparser-npm-config.rb cat integration_npm_config_env_var | while read line; do echo $line >> ~/.bashrc @@ -482,11 +484,11 @@ fi #Replace the placeholders in td-agent-bit.conf file for fluentbit with custom/default values in daemonset if [ ! -e "/etc/config/kube.conf" ] && [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then - /usr/bin/ruby2.7 td-agent-bit-conf-customizer.rb + ruby td-agent-bit-conf-customizer.rb fi #Parse the prometheus configmap to create a file with new custom settings. -/usr/bin/ruby2.7 tomlparser-prom-customconfig.rb +ruby tomlparser-prom-customconfig.rb #Setting default environment variables to be used in any case of failure in the above steps if [ ! -e "/etc/config/kube.conf" ]; then @@ -520,7 +522,7 @@ fi if [ ! -e "/etc/config/kube.conf" ]; then if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then #Parse the agent configmap to create a file with new custom settings. - /usr/bin/ruby2.7 tomlparser-prom-agent-config.rb + ruby tomlparser-prom-agent-config.rb #Sourcing config environment variable file if it exists if [ -e "side_car_fbit_config_env_var" ]; then cat side_car_fbit_config_env_var | while read line; do @@ -533,7 +535,7 @@ fi #Parse the configmap to set the right environment variables for MDM metrics configuration for Alerting. if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then - /usr/bin/ruby2.7 tomlparser-mdm-metrics-config.rb + ruby tomlparser-mdm-metrics-config.rb cat config_mdm_metrics_env_var | while read line; do echo $line >>~/.bashrc @@ -541,7 +543,7 @@ if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then source config_mdm_metrics_env_var #Parse the configmap to set the right environment variables for metric collection settings - /usr/bin/ruby2.7 tomlparser-metric-collection-config.rb + ruby tomlparser-metric-collection-config.rb cat config_metric_collection_env_var | while read line; do echo $line >>~/.bashrc @@ -552,7 +554,7 @@ fi # OSM scraping to be done in replicaset if sidecar car scraping is disabled and always do the scraping from the sidecar (It will always be either one of the two) if [[ ( ( ! -e "/etc/config/kube.conf" ) && ( "${CONTAINER_TYPE}" == "PrometheusSidecar" ) ) || ( ( -e "/etc/config/kube.conf" ) && ( "${SIDECAR_SCRAPING_ENABLED}" == "false" ) ) ]]; then - /usr/bin/ruby2.7 tomlparser-osm-config.rb + ruby tomlparser-osm-config.rb if [ -e "integration_osm_config_env_var" ]; then cat integration_osm_config_env_var | while read line; do @@ -649,7 +651,8 @@ if [ "$CONTAINER_RUNTIME" != "docker" ]; then fi echo "set caps for ruby process to read container env from proc" -sudo setcap cap_sys_ptrace,cap_dac_read_search+ep /usr/bin/ruby2.7 +RUBY_PATH=$(which ruby) +sudo setcap cap_sys_ptrace,cap_dac_read_search+ep "$RUBY_PATH" echo "export KUBELET_RUNTIME_OPERATIONS_METRIC="$KUBELET_RUNTIME_OPERATIONS_METRIC >> ~/.bashrc echo "export KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC="$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC >> ~/.bashrc diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 004fe7806..0e3e43757 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -1,3 +1,5 @@ +#!/bin/bash + TMPDIR="/opt" cd $TMPDIR @@ -11,6 +13,20 @@ fi #upgrade apt to latest version apt-get update && apt-get install -y apt && DEBIAN_FRONTEND=noninteractive apt-get install -y locales + +curl -sSL https://rvm.io/mpapis.asc | gpg --import - +curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - +curl -sSL https://get.rvm.io | bash -s stable + +# setup paths for ruby and rvm +if [ -f /etc/profile.d/rvm.sh ]; then + source /etc/profile.d/rvm.sh + echo "[ -f /etc/profile.d/rvm.sh ] && source /etc/profile.d/rvm.sh" >> ~/.bashrc +fi + +rvm install 3.1.1 +rvm --default use 3.1.1 + sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ update-locale LANG=en_US.UTF-8 @@ -57,15 +73,11 @@ sudo echo "deb https://packages.fluentbit.io/ubuntu/bionic bionic main" >> /etc/ sudo apt-get update sudo apt-get install td-agent-bit=1.7.8 -y -# install ruby2.7 -sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F5DA5F09C3173AA6 -sudo echo "deb http://ppa.launchpad.net/brightbox/ruby-ng/ubuntu bionic main" >> /etc/apt/sources.list -sudo apt-get update -sudo apt-get install ruby2.7 ruby2.7-dev gcc make -y # fluentd v1 gem -gem install fluentd -v "1.14.2" --no-document +gem install fluentd -v "1.14.6" --no-document fluentd --setup ./fluent gem install gyoku iso8601 --no-doc +gem install tomlrb -v "2.0.1" --no-document rm -f $TMPDIR/docker-cimprov*.sh @@ -75,7 +87,8 @@ rm -f $TMPDIR/envmdsd rm -f $TMPDIR/telegraf-*.tar.gz # remove build dependencies -sudo apt-get remove ruby2.7-dev gcc make -y +sudo apt-get remove gcc make -y +sudo apt autoremove -y # Remove settings for cron.daily that conflict with the node's cron.daily. Since both are trying to rotate the same files # in /var/log at the same time, the rotation doesn't happen correctly and then the *.1 file is forever logged to. diff --git a/kubernetes/windows/Dockerfile b/kubernetes/windows/Dockerfile index 3b663132e..019e9cda1 100644 --- a/kubernetes/windows/Dockerfile +++ b/kubernetes/windows/Dockerfile @@ -11,28 +11,27 @@ ARG IMAGE_TAG=win-ciprod06142022 # Docker creates a layer for every RUN-Statement RUN powershell -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" # Fluentd depends on cool.io whose fat gem is only available for Ruby < 2.5, so need to specify --platform ruby when install Ruby > 2.5 and install msys2 to get dev tools -RUN choco install -y ruby --version 2.7.5.1 --params "'/InstallDir:C:\ruby27'" \ -&& choco install -y msys2 --version 20211130.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby27\msys64'" \ +RUN choco install -y ruby --version 3.1.1.1 --params "'/InstallDir:C:\ruby31'" \ +&& choco install -y msys2 --version 20211130.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby31\msys64'" \ && choco install -y vim # gangams - optional MSYS2 update via ridk failing in merged docker file so skipping that since we dont need optional update RUN refreshenv \ && ridk install 3 \ && echo gem: --no-document >> C:\ProgramData\gemrc \ -&& gem install cool.io -v 1.5.4 --platform ruby \ +&& gem install cool.io -v 1.7.1 --platform ruby \ && gem install oj -v 3.3.10 \ -&& gem install json -v 2.2.0 \ -&& gem install fluentd -v 1.14.2 \ +&& gem install fluentd -v 1.14.6 \ && gem install win32-service -v 1.0.1 \ && gem install win32-ipc -v 0.7.0 \ && gem install win32-event -v 0.6.3 \ && gem install windows-pr -v 1.2.6 \ -&& gem install tomlrb -v 1.3.0 \ +&& gem install tomlrb -v 2.0.1 \ && gem install gyoku -v 1.3.1 \ && gem sources --clear-all # Remove gem cache and chocolatey -RUN powershell -Command "Remove-Item -Force C:\ruby27\lib\ruby\gems\2.7.0\cache\*.gem; Remove-Item -Recurse -Force 'C:\ProgramData\chocolatey'" +RUN powershell -Command "Remove-Item -Force C:\ruby31\lib\ruby\gems\3.1.0\cache\*.gem; Remove-Item -Recurse -Force 'C:\ProgramData\chocolatey'" SHELL ["powershell"] diff --git a/kubernetes/windows/Dockerfile-dev-base-image b/kubernetes/windows/Dockerfile-dev-base-image index 3aca6ae20..bcf28326e 100644 --- a/kubernetes/windows/Dockerfile-dev-base-image +++ b/kubernetes/windows/Dockerfile-dev-base-image @@ -8,8 +8,8 @@ LABEL vendor=Microsoft\ Corp \ # Docker creates a layer for every RUN-Statement RUN powershell -Command "Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))" # Fluentd depends on cool.io whose fat gem is only available for Ruby < 2.5, so need to specify --platform ruby when install Ruby > 2.5 and install msys2 to get dev tools -RUN choco install -y ruby --version 2.7.5.1 --params "'/InstallDir:C:\ruby27'" \ -&& choco install -y msys2 --version 20210604.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby27\msys64'" \ +RUN choco install -y ruby --version 3.1.1.1 --params "'/InstallDir:C:\ruby31'" \ +&& choco install -y msys2 --version 20210604.0.0 --params "'/NoPath /NoUpdate /InstallDir:C:\ruby31\msys64'" \ && choco install -y vim # gangams - optional MSYS2 update via ridk failing in merged docker file so skipping that since we dont need optional update @@ -29,7 +29,7 @@ RUN refreshenv \ && gem sources --clear-all # Remove gem cache and chocolatey -RUN powershell -Command "Remove-Item -Force C:\ruby27\lib\ruby\gems\2.7.0\cache\*.gem; Remove-Item -Recurse -Force 'C:\ProgramData\chocolatey'" +RUN powershell -Command "Remove-Item -Force C:\ruby31\lib\ruby\gems\3.1.0\cache\*.gem; Remove-Item -Recurse -Force 'C:\ProgramData\chocolatey'" SHELL ["powershell"] diff --git a/source/plugins/ruby/ApplicationInsightsUtility.rb b/source/plugins/ruby/ApplicationInsightsUtility.rb index 70d0a400e..b34cb20ee 100644 --- a/source/plugins/ruby/ApplicationInsightsUtility.rb +++ b/source/plugins/ruby/ApplicationInsightsUtility.rb @@ -7,7 +7,7 @@ class ApplicationInsightsUtility require_relative "DockerApiClient" require_relative "oms_common" require_relative "proxy_utils" - require "yajl/json_gem" + require "json" require "base64" @@HeartBeat = "HeartBeatEvent" diff --git a/source/plugins/ruby/CAdvisorMetricsAPIClient.rb b/source/plugins/ruby/CAdvisorMetricsAPIClient.rb index a0c50e6c5..b18e887fd 100644 --- a/source/plugins/ruby/CAdvisorMetricsAPIClient.rb +++ b/source/plugins/ruby/CAdvisorMetricsAPIClient.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true class CAdvisorMetricsAPIClient - require "yajl/json_gem" + require "json" require "logger" require "net/http" require "net/https" diff --git a/source/plugins/ruby/ContainerInventoryState.rb b/source/plugins/ruby/ContainerInventoryState.rb index 170fa65e3..7e5ca18e8 100644 --- a/source/plugins/ruby/ContainerInventoryState.rb +++ b/source/plugins/ruby/ContainerInventoryState.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true class ContainerInventoryState - require 'yajl/json_gem' + require 'json' require_relative 'omslog' @@InventoryDirectory = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory/" diff --git a/source/plugins/ruby/DockerApiClient.rb b/source/plugins/ruby/DockerApiClient.rb index 53dd1f39f..cff9f359f 100644 --- a/source/plugins/ruby/DockerApiClient.rb +++ b/source/plugins/ruby/DockerApiClient.rb @@ -3,7 +3,7 @@ class DockerApiClient require "socket" - require "yajl/json_gem" + require "json" require "timeout" require_relative "omslog" require_relative "DockerApiRestHelper" diff --git a/source/plugins/ruby/KubernetesApiClient.rb b/source/plugins/ruby/KubernetesApiClient.rb index ffd76bfbd..612035625 100644 --- a/source/plugins/ruby/KubernetesApiClient.rb +++ b/source/plugins/ruby/KubernetesApiClient.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true class KubernetesApiClient - require "yajl/json_gem" + require "json" require "logger" require "net/http" require "net/https" @@ -801,9 +801,9 @@ def getResourcesAndContinuationTokenV2(uri, api_group: nil) responseCode, resourceInfo = getKubeResourceInfoV2(uri, api_group: api_group) @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2 : Done getting resources from Kube API using url: #{uri} @ #{Time.now.utc.iso8601}" if !responseCode.nil? && responseCode == "200" && !resourceInfo.nil? - @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2:Start:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" - resourceInventory = Yajl::Parser.parse(StringIO.new(resourceInfo.body)) - @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2:End:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" + @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2:Start:Parsing data for #{uri} using JSON @ #{Time.now.utc.iso8601}" + resourceInventory = JSON.parse(resourceInfo.body) + @Log.info "KubernetesApiClient::getResourcesAndContinuationTokenV2:End:Parsing data for #{uri} using JSON @ #{Time.now.utc.iso8601}" resourceInfo = nil end if (!resourceInventory.nil? && !resourceInventory["metadata"].nil?) @@ -825,9 +825,9 @@ def getResourcesAndContinuationToken(uri, api_group: nil) resourceInfo = getKubeResourceInfo(uri, api_group: api_group) @Log.info "KubernetesApiClient::getResourcesAndContinuationToken : Done getting resources from Kube API using url: #{uri} @ #{Time.now.utc.iso8601}" if !resourceInfo.nil? - @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:Start:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" - resourceInventory = Yajl::Parser.parse(StringIO.new(resourceInfo.body)) - @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:End:Parsing data for #{uri} using yajl @ #{Time.now.utc.iso8601}" + @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:Start:Parsing data for #{uri} using JSON @ #{Time.now.utc.iso8601}" + resourceInventory = JSON.parse(resourceInfo.body) + @Log.info "KubernetesApiClient::getResourcesAndContinuationToken:End:Parsing data for #{uri} using JSON @ #{Time.now.utc.iso8601}" resourceInfo = nil end if (!resourceInventory.nil? && !resourceInventory["metadata"].nil?) diff --git a/source/plugins/ruby/MdmMetricsGenerator.rb b/source/plugins/ruby/MdmMetricsGenerator.rb index f4904697c..8ea06fe71 100644 --- a/source/plugins/ruby/MdmMetricsGenerator.rb +++ b/source/plugins/ruby/MdmMetricsGenerator.rb @@ -3,7 +3,6 @@ class MdmMetricsGenerator require "logger" - require "yajl/json_gem" require "json" require_relative "MdmAlertTemplates" require_relative "ApplicationInsightsUtility" @@ -140,7 +139,7 @@ def appendPodMetrics(records, metricName, metricHash, batch_time, metricsTemplat containerCountMetricValue: value, } end - records.push(Yajl::Parser.parse(StringIO.new(record))) + records.push(JSON.parse(record)) } else @log.info "No records found in hash for metric: #{metricName}" @@ -334,7 +333,7 @@ def getContainerResourceUtilMetricRecords(recordTimeStamp, metricName, percentag containerResourceUtilizationPercentage: percentageMetricValue, thresholdPercentageDimValue: thresholdPercentage, } - records.push(Yajl::Parser.parse(StringIO.new(resourceUtilRecord))) + records.push(JSON.parse(resourceUtilRecord)) # Adding another metric for threshold violation resourceThresholdViolatedRecord = MdmAlertTemplates::Container_resource_threshold_violation_template % { @@ -347,7 +346,7 @@ def getContainerResourceUtilMetricRecords(recordTimeStamp, metricName, percentag containerResourceThresholdViolated: isZeroFill ? 0 : 1, thresholdPercentageDimValue: thresholdPercentage, } - records.push(Yajl::Parser.parse(StringIO.new(resourceThresholdViolatedRecord))) + records.push(JSON.parse(resourceThresholdViolatedRecord)) rescue => errorStr @log.info "Error in getContainerResourceUtilMetricRecords: #{errorStr}" ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) @@ -374,7 +373,7 @@ def getPVResourceUtilMetricRecords(recordTimeStamp, metricName, computer, percen pvResourceUtilizationPercentage: percentageMetricValue, thresholdPercentageDimValue: thresholdPercentage, } - records.push(Yajl::Parser.parse(StringIO.new(resourceUtilRecord))) + records.push(JSON.parse(resourceUtilRecord)) # Adding another metric for threshold violation resourceThresholdViolatedRecord = MdmAlertTemplates::PV_resource_threshold_violation_template % { @@ -387,7 +386,7 @@ def getPVResourceUtilMetricRecords(recordTimeStamp, metricName, computer, percen pvResourceThresholdViolated: isZeroFill ? 0 : 1, thresholdPercentageDimValue: thresholdPercentage, } - records.push(Yajl::Parser.parse(StringIO.new(resourceThresholdViolatedRecord))) + records.push(JSON.parse(resourceThresholdViolatedRecord)) rescue => errorStr @log.info "Error in getPVResourceUtilMetricRecords: #{errorStr}" ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) @@ -418,7 +417,7 @@ def getDiskUsageMetricRecords(record) devicevalue: deviceName, diskUsagePercentageValue: usedPercent, } - records.push(Yajl::Parser.parse(StringIO.new(diskUsedPercentageRecord))) + records.push(JSON.parse(diskUsedPercentageRecord)) end rescue => errorStr @log.info "Error in getDiskUsageMetricRecords: #{errorStr}" @@ -469,7 +468,7 @@ def getMetricRecords(record) dimValues: dimValues, metricValue: v, } - records.push(Yajl::Parser.parse(StringIO.new(metricRecord))) + records.push(JSON.parse(metricRecord)) #@log.info "pushed mdmgenericmetric: #{k},#{v}" end } @@ -545,7 +544,7 @@ def getNodeResourceMetricRecords(record, metric_name, metric_value, percentage_m metricmaxvalue: metric_value, metricsumvalue: metric_value, } - records.push(Yajl::Parser.parse(StringIO.new(custommetricrecord))) + records.push(JSON.parse(custommetricrecord)) if !percentage_metric_value.nil? additional_record = MdmAlertTemplates::Node_resource_metrics_template % { @@ -558,7 +557,7 @@ def getNodeResourceMetricRecords(record, metric_name, metric_value, percentage_m metricmaxvalue: percentage_metric_value, metricsumvalue: percentage_metric_value, } - records.push(Yajl::Parser.parse(StringIO.new(additional_record))) + records.push(JSON.parse(additional_record)) end if !allocatable_percentage_metric_value.nil? @@ -572,7 +571,7 @@ def getNodeResourceMetricRecords(record, metric_name, metric_value, percentage_m metricmaxvalue: allocatable_percentage_metric_value, metricsumvalue: allocatable_percentage_metric_value, } - records.push(Yajl::Parser.parse(StringIO.new(additional_record))) + records.push(JSON.parse(additional_record)) end rescue => errorStr @log.info "Error in getNodeResourceMetricRecords: #{errorStr}" diff --git a/source/plugins/ruby/WatchStream.rb b/source/plugins/ruby/WatchStream.rb index 6cc850450..78ce25dd5 100644 --- a/source/plugins/ruby/WatchStream.rb +++ b/source/plugins/ruby/WatchStream.rb @@ -3,7 +3,7 @@ require "net/http" require "net/https" -require "yajl/json_gem" +require "json" require "logger" require "time" @@ -50,7 +50,7 @@ def each response.read_body do |chunk| buffer << chunk while (line = buffer.slice!(/.+\n/)) - yield(Yajl::Parser.parse(StringIO.new(line.chomp))) + yield(JSON.parse(line.chomp)) end end end diff --git a/source/plugins/ruby/arc_k8s_cluster_identity.rb b/source/plugins/ruby/arc_k8s_cluster_identity.rb index 39b8c1c96..43707b91f 100644 --- a/source/plugins/ruby/arc_k8s_cluster_identity.rb +++ b/source/plugins/ruby/arc_k8s_cluster_identity.rb @@ -3,7 +3,7 @@ require "net/http" require "net/https" require "uri" -require "yajl/json_gem" +require "json" require "base64" require "time" require_relative "KubernetesApiClient" diff --git a/source/plugins/ruby/filter_cadvisor2mdm.rb b/source/plugins/ruby/filter_cadvisor2mdm.rb index 6bafa372a..621c94992 100644 --- a/source/plugins/ruby/filter_cadvisor2mdm.rb +++ b/source/plugins/ruby/filter_cadvisor2mdm.rb @@ -6,7 +6,7 @@ module Fluent::Plugin require "logger" - require "yajl/json_gem" + require "json" require_relative "oms_common" require_relative "CustomMetricsUtils" require_relative "kubelet_utils" diff --git a/source/plugins/ruby/filter_inventory2mdm.rb b/source/plugins/ruby/filter_inventory2mdm.rb index 509ac608e..165bb63cf 100644 --- a/source/plugins/ruby/filter_inventory2mdm.rb +++ b/source/plugins/ruby/filter_inventory2mdm.rb @@ -6,7 +6,7 @@ module Fluent::Plugin require 'logger' - require 'yajl/json_gem' + require 'json' require_relative 'oms_common' require_relative 'CustomMetricsUtils' diff --git a/source/plugins/ruby/filter_telegraf2mdm.rb b/source/plugins/ruby/filter_telegraf2mdm.rb index fd71f1682..0819afdb7 100644 --- a/source/plugins/ruby/filter_telegraf2mdm.rb +++ b/source/plugins/ruby/filter_telegraf2mdm.rb @@ -6,7 +6,7 @@ module Fluent::Plugin require "logger" - require "yajl/json_gem" + require "json" require_relative "oms_common" require_relative "kubelet_utils" require_relative "MdmMetricsGenerator" diff --git a/source/plugins/ruby/in_cadvisor_perf.rb b/source/plugins/ruby/in_cadvisor_perf.rb index 901ecefab..d929e86fb 100644 --- a/source/plugins/ruby/in_cadvisor_perf.rb +++ b/source/plugins/ruby/in_cadvisor_perf.rb @@ -14,7 +14,7 @@ class CAdvisor_Perf_Input < Input def initialize super require "yaml" - require "yajl/json_gem" + require "json" require "time" require_relative "CAdvisorMetricsAPIClient" diff --git a/source/plugins/ruby/in_containerinventory.rb b/source/plugins/ruby/in_containerinventory.rb index c8ffe7d05..aeb70c68a 100644 --- a/source/plugins/ruby/in_containerinventory.rb +++ b/source/plugins/ruby/in_containerinventory.rb @@ -11,7 +11,7 @@ class Container_Inventory_Input < Input def initialize super - require "yajl/json_gem" + require "json" require "time" require_relative "ContainerInventoryState" require_relative "ApplicationInsightsUtility" diff --git a/source/plugins/ruby/in_kube_events.rb b/source/plugins/ruby/in_kube_events.rb index deeae6e14..6ccb02c54 100644 --- a/source/plugins/ruby/in_kube_events.rb +++ b/source/plugins/ruby/in_kube_events.rb @@ -10,8 +10,7 @@ class Kube_Event_Input < Input def initialize super - require "yajl/json_gem" - require "yajl" + require "json" require "time" require_relative "KubernetesApiClient" diff --git a/source/plugins/ruby/in_kube_nodes.rb b/source/plugins/ruby/in_kube_nodes.rb index 368eb61d4..8473cca81 100644 --- a/source/plugins/ruby/in_kube_nodes.rb +++ b/source/plugins/ruby/in_kube_nodes.rb @@ -16,8 +16,7 @@ def initialize(is_unit_test_mode = nil, kubernetesApiClient = nil, super() require "yaml" - require "yajl/json_gem" - require "yajl" + require "json" require "time" require_relative "KubernetesApiClient" diff --git a/source/plugins/ruby/in_kube_perfinventory.rb b/source/plugins/ruby/in_kube_perfinventory.rb index 20589167b..25f9c93e8 100644 --- a/source/plugins/ruby/in_kube_perfinventory.rb +++ b/source/plugins/ruby/in_kube_perfinventory.rb @@ -10,8 +10,7 @@ class Kube_PerfInventory_Input < Input def initialize super require "yaml" - require "yajl/json_gem" - require "yajl" + require "json" require "set" require "time" require "net/http" @@ -407,7 +406,7 @@ def getNodeAllocatableRecords() isAcquiredLock = f.flock(File::LOCK_EX | File::LOCK_NB) raise "in_kube_perfinventory:getNodeAllocatableRecords:Failed to acquire file lock @ #{Time.now.utc.iso8601}" if !isAcquiredLock startTime = (Time.now.to_f * 1000).to_i - nodeAllocatableRecords = Yajl::Parser.parse(f) + nodeAllocatableRecords = JSON.parse(f.read) timetakenMs = ((Time.now.to_f * 1000).to_i - startTime) $log.info "in_kube_perfinventory:getNodeAllocatableRecords:Number of Node Allocatable records: #{nodeAllocatableRecords.length} with time taken(ms) for read: #{timetakenMs} @ #{Time.now.utc.iso8601}" else diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 37c9741c3..a1986bd4a 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -12,8 +12,7 @@ class Kube_PodInventory_Input < Input def initialize super require "yaml" - require "yajl/json_gem" - require "yajl" + require "json" require "set" require "time" require "net/http" @@ -916,9 +915,9 @@ def watch_services else $log.info("in_kube_podinventory::watch_services: Done getting services from Kube API @ #{Time.now.utc.iso8601}") if !serviceInfo.nil? - $log.info("in_kube_podinventory::watch_services:Start:Parsing services data using yajl @ #{Time.now.utc.iso8601}") - serviceInventory = Yajl::Parser.parse(StringIO.new(serviceInfo.body)) - $log.info("in_kube_podinventory::watch_services:End:Parsing services data using yajl @ #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::watch_services:Start:Parsing services data using JSON @ #{Time.now.utc.iso8601}") + serviceInventory = JSON.parse(serviceInfo.body) + $log.info("in_kube_podinventory::watch_services:End:Parsing services data using JSON @ #{Time.now.utc.iso8601}") serviceInfo = nil if (!serviceInventory.nil? && !serviceInventory.empty?) servicesResourceVersion = serviceInventory["metadata"]["resourceVersion"] diff --git a/source/plugins/ruby/in_kube_podmdminventory.rb b/source/plugins/ruby/in_kube_podmdminventory.rb index b872650d2..38e07d860 100644 --- a/source/plugins/ruby/in_kube_podmdminventory.rb +++ b/source/plugins/ruby/in_kube_podmdminventory.rb @@ -14,8 +14,7 @@ class Kube_PodMDMInventory_Input < Input def initialize super require "yaml" - require "yajl/json_gem" - require "yajl" + require "json" require "set" require "time" require "net/http" @@ -187,7 +186,7 @@ def getMDMRecords() isAcquiredLock = f.flock(File::LOCK_EX | File::LOCK_NB) raise "in_kube_podmdminventory:getMDMRecords:Failed to acquire file lock @ #{Time.now.utc.iso8601}" if !isAcquiredLock startTime = (Time.now.to_f * 1000).to_i - mdmRecords = Yajl::Parser.parse(f) + mdmRecords = JSON.parse(f.read) timetakenMs = ((Time.now.to_f * 1000).to_i - startTime) if mdmRecords.nil? || mdmRecords.empty? || mdmRecords["items"].nil? || mdmRecords["collectionTime"] == @prevCollectionTime raise "in_kube_podmdminventory:getMDMRecords: either read mdmRecords is nil or empty or stale @ #{Time.now.utc.iso8601}" diff --git a/source/plugins/ruby/in_kube_pvinventory.rb b/source/plugins/ruby/in_kube_pvinventory.rb index fccfd459d..1e25e4057 100644 --- a/source/plugins/ruby/in_kube_pvinventory.rb +++ b/source/plugins/ruby/in_kube_pvinventory.rb @@ -12,8 +12,7 @@ class Kube_PVInventory_Input < Input def initialize super require "yaml" - require "yajl/json_gem" - require "yajl" + require "json" require "time" require_relative "KubernetesApiClient" require_relative "ApplicationInsightsUtility" diff --git a/source/plugins/ruby/in_kubestate_deployments.rb b/source/plugins/ruby/in_kubestate_deployments.rb index 0b563a890..92e6318b9 100644 --- a/source/plugins/ruby/in_kubestate_deployments.rb +++ b/source/plugins/ruby/in_kubestate_deployments.rb @@ -12,8 +12,7 @@ class Kube_Kubestate_Deployments_Input < Input def initialize super - require "yajl/json_gem" - require "yajl" + require "json" require "date" require "time" diff --git a/source/plugins/ruby/in_kubestate_hpa.rb b/source/plugins/ruby/in_kubestate_hpa.rb index 178f7944f..7f7e3aac5 100644 --- a/source/plugins/ruby/in_kubestate_hpa.rb +++ b/source/plugins/ruby/in_kubestate_hpa.rb @@ -10,8 +10,7 @@ class Kube_Kubestate_HPA_Input < Input def initialize super - require "yajl/json_gem" - require "yajl" + require "json" require "time" require_relative "KubernetesApiClient" diff --git a/source/plugins/ruby/in_win_cadvisor_perf.rb b/source/plugins/ruby/in_win_cadvisor_perf.rb index dd462fdf2..841c4867a 100644 --- a/source/plugins/ruby/in_win_cadvisor_perf.rb +++ b/source/plugins/ruby/in_win_cadvisor_perf.rb @@ -12,7 +12,7 @@ class Win_CAdvisor_Perf_Input < Input def initialize super require "yaml" - require "yajl/json_gem" + require "json" require "time" require_relative "CAdvisorMetricsAPIClient" diff --git a/source/plugins/ruby/kubelet_utils.rb b/source/plugins/ruby/kubelet_utils.rb index 368ca8639..b986f2ab7 100644 --- a/source/plugins/ruby/kubelet_utils.rb +++ b/source/plugins/ruby/kubelet_utils.rb @@ -3,7 +3,7 @@ # frozen_string_literal: true require "logger" -require "yajl/json_gem" +require "json" require_relative "CAdvisorMetricsAPIClient" require_relative "KubernetesApiClient" require "bigdecimal" @@ -52,7 +52,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) cpu_allocatable = 1.0 memory_allocatable = 1.0 - + allocatable_response = CAdvisorMetricsAPIClient.getCongifzCAdvisor(winNode: nil) parsed_response = JSON.parse(allocatable_response.body) @@ -66,7 +66,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::kubereserved_cpu: #{errorStr}" kubereserved_cpu = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::kubereserved_cpu: #{errorStr}") - end + end begin kubereserved_memory = parsed_response["kubeletconfig"]["kubeReserved"]["memory"] @@ -78,7 +78,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::kubereserved_memory: #{errorStr}" kubereserved_memory = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::kubereserved_memory: #{errorStr}") - end + end begin systemReserved_cpu = parsed_response["kubeletconfig"]["systemReserved"]["cpu"] if systemReserved_cpu.nil? || systemReserved_cpu == "" @@ -90,7 +90,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::systemReserved_cpu: #{errorStr}" systemReserved_cpu = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::systemReserved_cpu: #{errorStr}") - end + end begin explicitlyReserved_cpu = parsed_response["kubeletconfig"]["reservedCPUs"] @@ -103,19 +103,19 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::explicitlyReserved_cpu: #{errorStr}" explicitlyReserved_cpu = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::explicitlyReserved_cpu: #{errorStr}") - end + end begin - systemReserved_memory = parsed_response["kubeletconfig"]["systemReserved"]["memory"] - if systemReserved_memory.nil? || systemReserved_memory == "" + systemReserved_memory = parsed_response["kubeletconfig"]["systemReserved"]["memory"] + if systemReserved_memory.nil? || systemReserved_memory == "" systemReserved_memory = "0.0" - end - @log.info "get_node_allocatable::systemReserved_memory #{systemReserved_memory}" + end + @log.info "get_node_allocatable::systemReserved_memory #{systemReserved_memory}" rescue => errorStr - @log.error "Error in get_node_allocatable::systemReserved_memory: #{errorStr}" - systemReserved_memory = "0.0" + @log.error "Error in get_node_allocatable::systemReserved_memory: #{errorStr}" + systemReserved_memory = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::systemReserved_memory: #{errorStr}") - end + end begin evictionHard_memory = parsed_response["kubeletconfig"]["evictionHard"]["memory.available"] @@ -127,16 +127,16 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::evictionHard_memory: #{errorStr}" evictionHard_memory = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::evictionHard_memory: #{errorStr}") - end + end # do calculation in nanocore since that's what KubernetesApiClient.getMetricNumericValue expects cpu_capacity_number = cpu_capacity.to_i * 1000.0 ** 2 # subtract to get allocatable. Formula : Allocatable = Capacity - ( kube reserved + system reserved + eviction threshold ) # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable if KubernetesApiClient.getMetricNumericValue("cpu", explicitlyReserved_cpu) > 0 - cpu_allocatable = cpu_capacity_number - KubernetesApiClient.getMetricNumericValue("cpu", explicitlyReserved_cpu) + cpu_allocatable = cpu_capacity_number - KubernetesApiClient.getMetricNumericValue("cpu", explicitlyReserved_cpu) else - cpu_allocatable = cpu_capacity_number - (KubernetesApiClient.getMetricNumericValue("cpu", kubereserved_cpu) + KubernetesApiClient.getMetricNumericValue("cpu", systemReserved_cpu)) + cpu_allocatable = cpu_capacity_number - (KubernetesApiClient.getMetricNumericValue("cpu", kubereserved_cpu) + KubernetesApiClient.getMetricNumericValue("cpu", systemReserved_cpu)) end # convert back to units similar to what we get for capacity cpu_allocatable = cpu_allocatable / (1000.0 ** 2) @@ -165,7 +165,7 @@ def get_all_container_limits containerResourceDimensionHash = {} response = CAdvisorMetricsAPIClient.getPodsFromCAdvisor(winNode: nil) if !response.nil? && !response.body.nil? && !response.body.empty? - podInventory = Yajl::Parser.parse(StringIO.new(response.body)) + podInventory = JSON.parse(response.body) podInventory["items"].each do |items| @log.info "in pod inventory items..." podNameSpace = items["metadata"]["namespace"] diff --git a/source/plugins/ruby/kubernetes_container_inventory.rb b/source/plugins/ruby/kubernetes_container_inventory.rb index 81889b61b..3c2bda2e5 100644 --- a/source/plugins/ruby/kubernetes_container_inventory.rb +++ b/source/plugins/ruby/kubernetes_container_inventory.rb @@ -2,9 +2,8 @@ # frozen_string_literal: true class KubernetesContainerInventory - require "yajl/json_gem" - require "time" require "json" + require "time" require_relative "omslog" require_relative "ApplicationInsightsUtility" diff --git a/source/plugins/ruby/lib/application_insights/channel/contracts/json_serializable.rb b/source/plugins/ruby/lib/application_insights/channel/contracts/json_serializable.rb index 60838e215..8f4677044 100644 --- a/source/plugins/ruby/lib/application_insights/channel/contracts/json_serializable.rb +++ b/source/plugins/ruby/lib/application_insights/channel/contracts/json_serializable.rb @@ -1,4 +1,4 @@ -require 'yajl/json_gem' +require 'json' module ApplicationInsights module Channel diff --git a/source/plugins/ruby/lib/application_insights/channel/sender_base.rb b/source/plugins/ruby/lib/application_insights/channel/sender_base.rb index e5a4dea62..f5102c27a 100644 --- a/source/plugins/ruby/lib/application_insights/channel/sender_base.rb +++ b/source/plugins/ruby/lib/application_insights/channel/sender_base.rb @@ -1,4 +1,4 @@ -require "yajl/json_gem" +require "json" require "net/http" require "openssl" require "stringio" diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index 4561cdd9a..6fcc22cda 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -14,7 +14,7 @@ def initialize require "net/https" require "securerandom" require "uri" - require "yajl/json_gem" + require "json" require_relative "KubernetesApiClient" require_relative "ApplicationInsightsUtility" require_relative "constants" diff --git a/source/plugins/ruby/podinventory_to_mdm.rb b/source/plugins/ruby/podinventory_to_mdm.rb index a7f9c5435..5102274ed 100644 --- a/source/plugins/ruby/podinventory_to_mdm.rb +++ b/source/plugins/ruby/podinventory_to_mdm.rb @@ -3,7 +3,7 @@ # frozen_string_literal: true require "logger" -require "yajl/json_gem" +require "json" require "time" require_relative "oms_common" require_relative "CustomMetricsUtils" @@ -129,7 +129,7 @@ def get_pod_inventory_mdm_records(batch_time) controllerNameDimValue: podControllerNameDimValue, podCountMetricValue: value, } - records.push(Yajl::Parser.parse(record)) + records.push(JSON.parse(record)) } #Add pod metric records diff --git a/source/toml-parser/tomlrb.rb b/source/toml-parser/tomlrb.rb deleted file mode 100644 index c0eff9093..000000000 --- a/source/toml-parser/tomlrb.rb +++ /dev/null @@ -1,44 +0,0 @@ -require "time" -require "stringio" -require_relative "tomlrb/version" -require_relative "tomlrb/string_utils" -require_relative "tomlrb/scanner" -require_relative "tomlrb/parser" -require_relative "tomlrb/handler" - -module Tomlrb - class ParseError < StandardError; end - - # Parses a valid TOML string into its Ruby data structure - # - # @param string_or_io [String, StringIO] the content - # @param options [Hash] the options hash - # @option options [Boolean] :symbolize_keys (false) whether to return the keys as symbols or strings - # @return [Hash] the Ruby data structure represented by the input - def self.parse(string_or_io, **options) - io = string_or_io.is_a?(String) ? StringIO.new(string_or_io) : string_or_io - scanner = Scanner.new(io) - parser = Parser.new(scanner, options) - begin - handler = parser.parse - rescue Racc::ParseError => e - raise ParseError, e.message - end - - handler.output - end - - # Reads a file content and parses it into its Ruby data structure - # - # @param path [String] the path to the file - # @param options [Hash] the options hash - # @option options [Boolean] :symbolize_keys (false) whether to return the keys as symbols or strings - # @return [Hash] the Ruby data structure represented by the input - def self.load_file(path, **options) - # By default Ruby sets the external encoding of an IO object to the - # default external encoding. The default external encoding is set by - # locale encoding or the interpreter -E option. - tmp = File.read(path, :encoding => "utf-8") - Tomlrb.parse(tmp, options) - end -end diff --git a/source/toml-parser/tomlrb/generated_parser.rb b/source/toml-parser/tomlrb/generated_parser.rb deleted file mode 100644 index ebf815e7d..000000000 --- a/source/toml-parser/tomlrb/generated_parser.rb +++ /dev/null @@ -1,542 +0,0 @@ -# -# DO NOT MODIFY!!!! -# This file is automatically generated by Racc 1.4.14 -# from Racc grammer file "". -# - -require 'racc/parser.rb' -module Tomlrb - class GeneratedParser < Racc::Parser -##### State transition tables begin ### - -racc_action_table = [ - 2, 17, 11, 31, 12, 31, 13, 27, 14, 77, - 15, 16, 8, 78, 32, 10, 33, 29, 34, 29, - 57, 58, 59, 60, 56, 53, 52, 54, 55, 46, - 40, 41, 10, 57, 58, 59, 60, 56, 53, 52, - 54, 55, 46, 69, 70, 10, 57, 58, 59, 60, - 56, 53, 52, 54, 55, 46, 35, 36, 10, 57, - 58, 59, 60, 56, 53, 52, 54, 55, 46, 37, - 38, 10, 57, 58, 59, 60, 56, 53, 52, 54, - 55, 46, 43, 66, 10, 57, 58, 59, 60, 56, - 53, 52, 54, 55, 46, nil, nil, 10, 57, 58, - 59, 60, 56, 53, 52, 54, 55, 46, nil, nil, - 10, 57, 58, 59, 60, 56, 53, 52, 54, 55, - 46, 73, nil, 10, 57, 58, 59, 60, 56, 53, - 52, 54, 55, 46, 73, 21, 10, 22, nil, 23, - nil, 24, nil, 25, 26, 21, 19, 22, nil, 23, - nil, 24, nil, 25, 26, nil, 19 ] - -racc_action_check = [ - 1, 2, 1, 9, 1, 70, 1, 8, 1, 74, - 1, 1, 1, 74, 11, 1, 12, 9, 13, 70, - 32, 32, 32, 32, 32, 32, 32, 32, 32, 32, - 20, 20, 32, 33, 33, 33, 33, 33, 33, 33, - 33, 33, 33, 42, 42, 33, 34, 34, 34, 34, - 34, 34, 34, 34, 34, 34, 14, 15, 34, 35, - 35, 35, 35, 35, 35, 35, 35, 35, 35, 16, - 19, 35, 36, 36, 36, 36, 36, 36, 36, 36, - 36, 36, 30, 40, 36, 37, 37, 37, 37, 37, - 37, 37, 37, 37, 37, nil, nil, 37, 43, 43, - 43, 43, 43, 43, 43, 43, 43, 43, nil, nil, - 43, 45, 45, 45, 45, 45, 45, 45, 45, 45, - 45, 45, nil, 45, 78, 78, 78, 78, 78, 78, - 78, 78, 78, 78, 78, 7, 78, 7, nil, 7, - nil, 7, nil, 7, 7, 41, 7, 41, nil, 41, - nil, 41, nil, 41, 41, nil, 41 ] - -racc_action_pointer = [ - nil, 0, 1, nil, nil, nil, nil, 133, -5, 1, - nil, -4, -2, 0, 38, 39, 51, nil, nil, 57, - 17, nil, nil, nil, nil, nil, nil, nil, nil, nil, - 64, nil, 17, 30, 43, 56, 69, 82, nil, nil, - 70, 143, 27, 95, nil, 108, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - 3, nil, nil, nil, -4, nil, nil, nil, 121, nil ] - -racc_action_default = [ - -1, -56, -56, -2, -3, -4, -5, -56, -8, -56, - -22, -56, -56, -56, -56, -56, -56, 80, -6, -10, - -56, -15, -16, -17, -18, -19, -20, -7, -21, -23, - -56, -27, -46, -46, -46, -46, -46, -46, -9, -11, - -13, -56, -56, -46, -29, -46, -40, -41, -42, -43, - -44, -45, -47, -48, -49, -50, -51, -52, -53, -54, - -55, -30, -31, -32, -33, -34, -12, -14, -24, -25, - -56, -28, -35, -36, -56, -26, -37, -38, -46, -39 ] - -racc_goto_table = [ - 28, 18, 1, 72, 44, 61, 62, 63, 64, 65, - 3, 4, 5, 6, 7, 71, 39, 42, 68, 76, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, 67, 79, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 75 ] - -racc_goto_check = [ - 11, 7, 1, 18, 15, 15, 15, 15, 15, 15, - 2, 3, 4, 5, 6, 15, 9, 13, 14, 19, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, 7, 18, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, - nil, 11 ] - -racc_goto_pointer = [ - nil, 2, 9, 10, 11, 12, 13, -6, nil, -4, - nil, -9, nil, -13, -24, -28, nil, nil, -42, -55, - nil, nil, nil ] - -racc_goto_default = [ - nil, nil, nil, nil, nil, 49, nil, nil, 20, nil, - 9, nil, 30, nil, nil, 74, 48, 45, nil, nil, - 47, 50, 51 ] - -racc_reduce_table = [ - 0, 0, :racc_error, - 0, 20, :_reduce_none, - 2, 20, :_reduce_none, - 1, 21, :_reduce_none, - 1, 21, :_reduce_none, - 1, 21, :_reduce_none, - 2, 22, :_reduce_none, - 2, 25, :_reduce_7, - 1, 25, :_reduce_8, - 2, 26, :_reduce_9, - 1, 26, :_reduce_10, - 2, 26, :_reduce_none, - 2, 28, :_reduce_12, - 1, 28, :_reduce_13, - 2, 28, :_reduce_none, - 1, 27, :_reduce_15, - 1, 27, :_reduce_16, - 1, 27, :_reduce_17, - 1, 27, :_reduce_18, - 1, 27, :_reduce_19, - 1, 27, :_reduce_20, - 2, 24, :_reduce_none, - 1, 29, :_reduce_22, - 1, 30, :_reduce_23, - 3, 30, :_reduce_none, - 1, 33, :_reduce_25, - 2, 33, :_reduce_none, - 1, 31, :_reduce_27, - 2, 32, :_reduce_none, - 3, 23, :_reduce_29, - 3, 23, :_reduce_30, - 3, 23, :_reduce_31, - 3, 23, :_reduce_32, - 3, 23, :_reduce_33, - 3, 23, :_reduce_34, - 2, 35, :_reduce_none, - 1, 37, :_reduce_36, - 2, 37, :_reduce_none, - 1, 38, :_reduce_38, - 2, 38, :_reduce_none, - 1, 36, :_reduce_40, - 1, 34, :_reduce_41, - 1, 34, :_reduce_none, - 1, 34, :_reduce_none, - 1, 39, :_reduce_none, - 1, 39, :_reduce_none, - 0, 41, :_reduce_none, - 1, 41, :_reduce_47, - 1, 41, :_reduce_48, - 1, 41, :_reduce_49, - 1, 41, :_reduce_50, - 1, 41, :_reduce_51, - 1, 40, :_reduce_52, - 1, 40, :_reduce_53, - 1, 40, :_reduce_54, - 1, 40, :_reduce_55 ] - -racc_reduce_n = 56 - -racc_shift_n = 80 - -racc_token_table = { - false => 0, - :error => 1, - :IDENTIFIER => 2, - :STRING_MULTI => 3, - :STRING_BASIC => 4, - :STRING_LITERAL_MULTI => 5, - :STRING_LITERAL => 6, - :DATETIME => 7, - :INTEGER => 8, - :FLOAT => 9, - :TRUE => 10, - :FALSE => 11, - "[" => 12, - "]" => 13, - "." => 14, - "{" => 15, - "}" => 16, - "," => 17, - "=" => 18 } - -racc_nt_base = 19 - -racc_use_result_var = true - -Racc_arg = [ - racc_action_table, - racc_action_check, - racc_action_default, - racc_action_pointer, - racc_goto_table, - racc_goto_check, - racc_goto_default, - racc_goto_pointer, - racc_nt_base, - racc_reduce_table, - racc_token_table, - racc_shift_n, - racc_reduce_n, - racc_use_result_var ] - -Racc_token_to_s_table = [ - "$end", - "error", - "IDENTIFIER", - "STRING_MULTI", - "STRING_BASIC", - "STRING_LITERAL_MULTI", - "STRING_LITERAL", - "DATETIME", - "INTEGER", - "FLOAT", - "TRUE", - "FALSE", - "\"[\"", - "\"]\"", - "\".\"", - "\"{\"", - "\"}\"", - "\",\"", - "\"=\"", - "$start", - "expressions", - "expression", - "table", - "assignment", - "inline_table", - "table_start", - "table_continued", - "table_identifier", - "table_next", - "inline_table_start", - "inline_continued", - "inline_assignment_key", - "inline_assignment_value", - "inline_next", - "value", - "array", - "start_array", - "array_continued", - "array_next", - "scalar", - "string", - "literal" ] - -Racc_debug_parser = false - -##### State transition tables end ##### - -# reduce 0 omitted - -# reduce 1 omitted - -# reduce 2 omitted - -# reduce 3 omitted - -# reduce 4 omitted - -# reduce 5 omitted - -# reduce 6 omitted - -module_eval(<<'.,.,', 'parser.y', 15) - def _reduce_7(val, _values, result) - @handler.start_(:array_of_tables) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 16) - def _reduce_8(val, _values, result) - @handler.start_(:table) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 19) - def _reduce_9(val, _values, result) - array = @handler.end_(:array_of_tables); @handler.set_context(array, is_array_of_tables: true) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 20) - def _reduce_10(val, _values, result) - array = @handler.end_(:table); @handler.set_context(array) - result - end -.,., - -# reduce 11 omitted - -module_eval(<<'.,.,', 'parser.y', 24) - def _reduce_12(val, _values, result) - array = @handler.end_(:array_of_tables); @handler.set_context(array, is_array_of_tables: true) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 25) - def _reduce_13(val, _values, result) - array = @handler.end_(:table); @handler.set_context(array) - result - end -.,., - -# reduce 14 omitted - -module_eval(<<'.,.,', 'parser.y', 29) - def _reduce_15(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 30) - def _reduce_16(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 31) - def _reduce_17(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 32) - def _reduce_18(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 33) - def _reduce_19(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 34) - def _reduce_20(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -# reduce 21 omitted - -module_eval(<<'.,.,', 'parser.y', 40) - def _reduce_22(val, _values, result) - @handler.start_(:inline) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 43) - def _reduce_23(val, _values, result) - array = @handler.end_(:inline); @handler.push(Hash[*array]) - result - end -.,., - -# reduce 24 omitted - -module_eval(<<'.,.,', 'parser.y', 48) - def _reduce_25(val, _values, result) - array = @handler.end_(:inline) - array.map!.with_index{ |n,i| i.even? ? n.to_sym : n } if @handler.symbolize_keys - @handler.push(Hash[*array]) - - result - end -.,., - -# reduce 26 omitted - -module_eval(<<'.,.,', 'parser.y', 55) - def _reduce_27(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -# reduce 28 omitted - -module_eval(<<'.,.,', 'parser.y', 61) - def _reduce_29(val, _values, result) - @handler.assign(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 62) - def _reduce_30(val, _values, result) - @handler.assign(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 63) - def _reduce_31(val, _values, result) - @handler.assign(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 64) - def _reduce_32(val, _values, result) - @handler.assign(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 65) - def _reduce_33(val, _values, result) - @handler.assign(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 66) - def _reduce_34(val, _values, result) - @handler.assign(val[0]) - result - end -.,., - -# reduce 35 omitted - -module_eval(<<'.,.,', 'parser.y', 72) - def _reduce_36(val, _values, result) - array = @handler.end_(:array); @handler.push(array) - result - end -.,., - -# reduce 37 omitted - -module_eval(<<'.,.,', 'parser.y', 76) - def _reduce_38(val, _values, result) - array = @handler.end_(:array); @handler.push(array) - result - end -.,., - -# reduce 39 omitted - -module_eval(<<'.,.,', 'parser.y', 80) - def _reduce_40(val, _values, result) - @handler.start_(:array) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 83) - def _reduce_41(val, _values, result) - @handler.push(val[0]) - result - end -.,., - -# reduce 42 omitted - -# reduce 43 omitted - -# reduce 44 omitted - -# reduce 45 omitted - -# reduce 46 omitted - -module_eval(<<'.,.,', 'parser.y', 92) - def _reduce_47(val, _values, result) - result = val[0].to_f - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 93) - def _reduce_48(val, _values, result) - result = val[0].to_i - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 94) - def _reduce_49(val, _values, result) - result = true - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 95) - def _reduce_50(val, _values, result) - result = false - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 96) - def _reduce_51(val, _values, result) - result = Time.new(*val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 99) - def _reduce_52(val, _values, result) - result = StringUtils.replace_escaped_chars(StringUtils.multiline_replacements(val[0])) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 100) - def _reduce_53(val, _values, result) - result = StringUtils.replace_escaped_chars(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 101) - def _reduce_54(val, _values, result) - result = StringUtils.strip_spaces(val[0]) - result - end -.,., - -module_eval(<<'.,.,', 'parser.y', 102) - def _reduce_55(val, _values, result) - result = val[0] - result - end -.,., - -def _reduce_none(val, _values, result) - val[0] -end - - end # class GeneratedParser - end # module Tomlrb diff --git a/source/toml-parser/tomlrb/handler.rb b/source/toml-parser/tomlrb/handler.rb deleted file mode 100644 index d60b54bc3..000000000 --- a/source/toml-parser/tomlrb/handler.rb +++ /dev/null @@ -1,73 +0,0 @@ -module Tomlrb - class Handler - attr_reader :output, :symbolize_keys - - def initialize(**options) - @output = {} - @current = @output - @stack = [] - @array_names = [] - @symbolize_keys = options[:symbolize_keys] - end - - def set_context(identifiers, is_array_of_tables: false) - @current = @output - - deal_with_array_of_tables(identifiers, is_array_of_tables) do |identifierz| - identifierz.each do |k| - k = k.to_sym if @symbolize_keys - if @current[k].is_a?(Array) - @current[k] << {} if @current[k].empty? - @current = @current[k].last - else - @current[k] ||= {} - @current = @current[k] - end - end - end - end - - def deal_with_array_of_tables(identifiers, is_array_of_tables) - identifiers.map!{|n| n.gsub("\"", '')} - stringified_identifier = identifiers.join('.') - - if is_array_of_tables - @array_names << stringified_identifier - last_identifier = identifiers.pop - elsif @array_names.include?(stringified_identifier) - raise ParseError, 'Cannot define a normal table with the same name as an already established array' - end - - yield(identifiers) - - if is_array_of_tables - last_identifier = last_identifier.to_sym if @symbolize_keys - @current[last_identifier] ||= [] - @current[last_identifier] << {} - @current = @current[last_identifier].last - end - end - - def assign(k) - k = k.to_sym if @symbolize_keys - @current[k] = @stack.pop - end - - def push(o) - @stack << o - end - - def start_(type) - push([type]) - end - - def end_(type) - array = [] - while (value = @stack.pop) != [type] - raise ParseError, 'Unclosed table' unless value - array.unshift(value) - end - array - end - end -end diff --git a/source/toml-parser/tomlrb/parser.rb b/source/toml-parser/tomlrb/parser.rb deleted file mode 100644 index 31771a1ca..000000000 --- a/source/toml-parser/tomlrb/parser.rb +++ /dev/null @@ -1,18 +0,0 @@ -require_relative "generated_parser" - -class Tomlrb::Parser < Tomlrb::GeneratedParser - def initialize(tokenizer, **options) - @tokenizer = tokenizer - @handler = Tomlrb::Handler.new(options) - super() - end - - def next_token - @tokenizer.next_token - end - - def parse - do_parse - @handler - end -end diff --git a/source/toml-parser/tomlrb/parser.y b/source/toml-parser/tomlrb/parser.y deleted file mode 100644 index fcebcac06..000000000 --- a/source/toml-parser/tomlrb/parser.y +++ /dev/null @@ -1,104 +0,0 @@ -class Tomlrb::GeneratedParser -token IDENTIFIER STRING_MULTI STRING_BASIC STRING_LITERAL_MULTI STRING_LITERAL DATETIME INTEGER FLOAT TRUE FALSE -rule - expressions - | expressions expression - ; - expression - : table - | assignment - | inline_table - ; - table - : table_start table_continued - ; - table_start - : '[' '[' { @handler.start_(:array_of_tables) } - | '[' { @handler.start_(:table) } - ; - table_continued - : ']' ']' { array = @handler.end_(:array_of_tables); @handler.set_context(array, is_array_of_tables: true) } - | ']' { array = @handler.end_(:table); @handler.set_context(array) } - | table_identifier table_next - ; - table_next - : ']' ']' { array = @handler.end_(:array_of_tables); @handler.set_context(array, is_array_of_tables: true) } - | ']' { array = @handler.end_(:table); @handler.set_context(array) } - | '.' table_continued - ; - table_identifier - : IDENTIFIER { @handler.push(val[0]) } - | STRING_BASIC { @handler.push(val[0]) } - | STRING_LITERAL { @handler.push(val[0]) } - | INTEGER { @handler.push(val[0]) } - | TRUE { @handler.push(val[0]) } - | FALSE { @handler.push(val[0]) } - ; - inline_table - : inline_table_start inline_continued - ; - inline_table_start - : '{' { @handler.start_(:inline) } - ; - inline_continued - : '}' { array = @handler.end_(:inline); @handler.push(Hash[*array]) } - | inline_assignment_key inline_assignment_value inline_next - ; - inline_next - : '}' { - array = @handler.end_(:inline) - array.map!.with_index{ |n,i| i.even? ? n.to_sym : n } if @handler.symbolize_keys - @handler.push(Hash[*array]) - } - | ',' inline_continued - ; - inline_assignment_key - : IDENTIFIER { @handler.push(val[0]) } - ; - inline_assignment_value - : '=' value - ; - assignment - : IDENTIFIER '=' value { @handler.assign(val[0]) } - | STRING_BASIC '=' value { @handler.assign(val[0]) } - | STRING_LITERAL '=' value { @handler.assign(val[0]) } - | INTEGER '=' value { @handler.assign(val[0]) } - | TRUE '=' value { @handler.assign(val[0]) } - | FALSE '=' value { @handler.assign(val[0]) } - ; - array - : start_array array_continued - ; - array_continued - : ']' { array = @handler.end_(:array); @handler.push(array) } - | value array_next - ; - array_next - : ']' { array = @handler.end_(:array); @handler.push(array) } - | ',' array_continued - ; - start_array - : '[' { @handler.start_(:array) } - ; - value - : scalar { @handler.push(val[0]) } - | array - | inline_table - ; - scalar - : string - | literal - ; - literal - | FLOAT { result = val[0].to_f } - | INTEGER { result = val[0].to_i } - | TRUE { result = true } - | FALSE { result = false } - | DATETIME { result = Time.new(*val[0])} - ; - string - : STRING_MULTI { result = StringUtils.replace_escaped_chars(StringUtils.multiline_replacements(val[0])) } - | STRING_BASIC { result = StringUtils.replace_escaped_chars(val[0]) } - | STRING_LITERAL_MULTI { result = StringUtils.strip_spaces(val[0]) } - | STRING_LITERAL { result = val[0] } - ; diff --git a/source/toml-parser/tomlrb/scanner.rb b/source/toml-parser/tomlrb/scanner.rb deleted file mode 100644 index d0f479eef..000000000 --- a/source/toml-parser/tomlrb/scanner.rb +++ /dev/null @@ -1,54 +0,0 @@ -require 'strscan' - -module Tomlrb - class Scanner - COMMENT = /#.*/ - IDENTIFIER = /[A-Za-z0-9_-]+/ - SPACE = /[ \t\r\n]/ - STRING_BASIC = /(["])(?:\\?.)*?\1/ - STRING_MULTI = /"{3}([\s\S]*?"{3,4})/m - STRING_LITERAL = /(['])(?:\\?.)*?\1/ - STRING_LITERAL_MULTI = /'{3}([\s\S]*?'{3})/m - DATETIME = /(-?\d{4})-(\d{2})-(\d{2})(?:(?:t|\s)(\d{2}):(\d{2}):(\d{2}(?:\.\d+)?))?(z|[-+]\d{2}:\d{2})?/i - FLOAT = /[+-]?(?:[0-9_]+\.[0-9_]*|\d+(?=[eE]))(?:[eE][+-]?[0-9_]+)?/ - INTEGER = /[+-]?([1-9](_?\d)*|0)(?![A-Za-z0-9_-]+)/ - TRUE = /true/ - FALSE = /false/ - - def initialize(io) - @ss = StringScanner.new(io.read) - end - - def next_token - return if @ss.eos? - - case - when @ss.scan(SPACE) then next_token - when @ss.scan(COMMENT) then next_token - when @ss.scan(DATETIME) then process_datetime - when text = @ss.scan(STRING_MULTI) then [:STRING_MULTI, text[3..-4]] - when text = @ss.scan(STRING_BASIC) then [:STRING_BASIC, text[1..-2]] - when text = @ss.scan(STRING_LITERAL_MULTI) then [:STRING_LITERAL_MULTI, text[3..-4]] - when text = @ss.scan(STRING_LITERAL) then [:STRING_LITERAL, text[1..-2]] - when text = @ss.scan(FLOAT) then [:FLOAT, text] - when text = @ss.scan(INTEGER) then [:INTEGER, text] - when text = @ss.scan(TRUE) then [:TRUE, text] - when text = @ss.scan(FALSE) then [:FALSE, text] - when text = @ss.scan(IDENTIFIER) then [:IDENTIFIER, text] - else - x = @ss.getch - [x, x] - end - end - - def process_datetime - if @ss[7].nil? - offset = '+00:00' - else - offset = @ss[7].gsub('Z', '+00:00') - end - args = [@ss[1], @ss[2], @ss[3], @ss[4] || 0, @ss[5] || 0, @ss[6].to_f, offset] - [:DATETIME, args] - end - end -end diff --git a/source/toml-parser/tomlrb/string_utils.rb b/source/toml-parser/tomlrb/string_utils.rb deleted file mode 100644 index 53d27e414..000000000 --- a/source/toml-parser/tomlrb/string_utils.rb +++ /dev/null @@ -1,33 +0,0 @@ -module Tomlrb - class StringUtils - - SPECIAL_CHARS = { - '\\t' => "\t", - '\\b' => "\b", - '\\f' => "\f", - '\\n' => "\n", - '\\r' => "\r", - '\\"' => '"', - '\\\\' => '\\' - }.freeze - - def self.multiline_replacements(str) - strip_spaces(str).gsub(/\\\n\s+/, '') - end - - def self.replace_escaped_chars(str) - str.gsub(/\\(u[\da-fA-F]{4}|U[\da-fA-F]{8}|.)/) do |m| - if m.size == 2 - SPECIAL_CHARS[m] || (raise Tomlrb::ParseError.new "Escape sequence #{m} is reserved") - else - m[2..-1].to_i(16).chr(Encoding::UTF_8) - end - end - end - - def self.strip_spaces(str) - str[0] = '' if str[0] == "\n" - str - end - end -end diff --git a/source/toml-parser/tomlrb/version.rb b/source/toml-parser/tomlrb/version.rb deleted file mode 100644 index b72a81b60..000000000 --- a/source/toml-parser/tomlrb/version.rb +++ /dev/null @@ -1,3 +0,0 @@ -module Tomlrb - VERSION = "1.2.8" -end