diff --git a/.github/workflows/pr-checker.yml b/.github/workflows/pr-checker.yml index 8334349e2..f0cea063d 100644 --- a/.github/workflows/pr-checker.yml +++ b/.github/workflows/pr-checker.yml @@ -56,7 +56,8 @@ jobs: format: 'table' severity: 'CRITICAL,HIGH' vuln-type: 'os,library' - skip-files: '/usr/sbin/telegraf,/opt/telegraf,/opt/td-agent-bit/bin/out_oms.so' + #[vishwa] - Fix telegraf & test all for next release - see work item #https://msazure.visualstudio.com/InfrastructureInsights/_workitems/edit/13322134 + skip-files: '/usr/sbin/telegraf,/opt/telegraf' exit-code: '1' timeout: '5m0s' ignore-unfixed: true @@ -95,3 +96,4 @@ jobs: cd ./kubernetes/windows/ && docker build . --file Dockerfile -t $env:IMAGETAG --build-arg IMAGE_TAG=$env:IMAGETAG_TELEMETRY - name: List-docker-images run: docker images --digests --all + diff --git a/.pipelines/azure_pipeline_dev.yaml b/.pipelines/azure_pipeline_dev.yaml new file mode 100644 index 000000000..ba8a530fc --- /dev/null +++ b/.pipelines/azure_pipeline_dev.yaml @@ -0,0 +1,104 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- ci_dev + +pool: + name: Azure-Pipelines-CI-Test-EO + +variables: + armServiceConnectionName: 'ci-1es-acr-connection' + subscription: '9b96ebbd-c57a-42d1-bbe9-b69296e4c7fb' + containerRegistry: 'containerinsightsprod' + repoImageName: '${{ variables.containerRegistry }}.azurecr.io/public/azuremonitor/containerinsights/cidev' + +steps: +- bash: | + commit=$(git rev-parse --short HEAD) + echo "##vso[task.setvariable variable=commit;]$commit" + + datetime=$(date +'%Y%m%d%s') + echo "##vso[task.setvariable variable=datetime;]$datetime" + + cd $(Build.SourcesDirectory)/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz pushAgentToAcr.sh + + cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ pushChartToAcr.sh + +- task: CopyFiles@2 + displayName: "Copy ev2 deployment artifacts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/deployment" + Contents: | + **/* + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + +- task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/.pipelines" + Contents: | + *.sh + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + +- task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/kubernetes" + Contents: | + *.yaml + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + +- task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/charts" + Contents: | + **/* + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + +- task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/test/e2e" + Contents: | + *.yaml + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + +- task: AzureCLI@2 + displayName: "Docker multi-arch linux build" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + + sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + docker buildx create --name testbuilder + docker buildx use testbuilder + + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + + docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageName }}:$(datetime)-$(commit) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/metadata.json --build-arg IMAGE_TAG=$(datetime)-$(commit) --push . + + docker pull ${{ variables.repoImageName }}:$(datetime)-$(commit) + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)' + DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04, ${{ variables.repoImageName }}:$(datetime)-$(commit)' + +- task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: drop diff --git a/.pipelines/azure_pipeline_prod.yaml b/.pipelines/azure_pipeline_prod.yaml new file mode 100644 index 000000000..6f5d8bd45 --- /dev/null +++ b/.pipelines/azure_pipeline_prod.yaml @@ -0,0 +1,104 @@ +# Starter pipeline +# Start with a minimal pipeline that you can customize to build and deploy your code. +# Add steps that build, run tests, deploy, and more: +# https://aka.ms/yaml + +trigger: +- ci_prod + +pool: + name: Azure-Pipelines-CI-Prod-EO + +variables: + armServiceConnectionName: 'ci-1es-acr-connection-prod' + subscription: '30c56c3a-54da-46ea-b004-06eb33432687' + containerRegistry: 'containerinsightsbuild' + repoImageName: '${{ variables.containerRegistry }}.azurecr.io/official/linux' + +steps: +- bash: | + commit=$(git rev-parse --short HEAD) + echo "##vso[task.setvariable variable=commit;]$commit" + + datetime=$(date +'%Y%m%d%s') + echo "##vso[task.setvariable variable=datetime;]$datetime" + + cd $(Build.SourcesDirectory)/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz pushAgentToAcr.sh + + cd $(Build.SourcesDirectory)/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts + tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ pushChartToAcr.sh + +- task: CopyFiles@2 + displayName: "Copy ev2 deployment artifacts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/deployment" + Contents: | + **/* + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + +- task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/.pipelines" + Contents: | + **/*.sh + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + +- task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/kubernetes" + Contents: | + *.yaml + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + +- task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/charts" + Contents: | + **/* + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + +- task: CopyFiles@2 + displayName: "Copy ev2 deployment scripts" + inputs: + SourceFolder: "$(Build.SourcesDirectory)/test/e2e" + Contents: | + *.yaml + TargetFolder: '$(Build.ArtifactStagingDirectory)/build' + +- task: AzureCLI@2 + displayName: "Docker multi-arch linux build" + inputs: + azureSubscription: ${{ variables.armServiceConnectionName }} + scriptType: bash + scriptLocation: inlineScript + inlineScript: | + + sudo apt-get update && sudo apt-get -y install qemu binfmt-support qemu-user-static + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + docker buildx create --name testbuilder + docker buildx use testbuilder + + az --version + az account show + az account set -s ${{ variables.subscription }} + az acr login -n ${{ variables.containerRegistry }} + + docker buildx build --platform linux/amd64,linux/arm64 --tag ${{ variables.repoImageName }}:ciprod-$(datetime)-$(commit) -f kubernetes/linux/Dockerfile.multiarch --metadata-file $(Build.ArtifactStagingDirectory)/metadata.json --build-arg IMAGE_TAG=ciprod-$(datetime)-$(commit) --push . + + docker pull ${{ variables.repoImageName }}:ciprod-$(datetime)-$(commit) + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'Generation Task' + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)' + DockerImagesToScan: 'golang:1.15.14, ubuntu:18.04, ${{ variables.repoImageName }}:ciprod-$(datetime)-$(commit)' + +- task: PublishBuildArtifacts@1 + inputs: + pathToPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: drop \ No newline at end of file diff --git a/.pipelines/build-linux.sh b/.pipelines/build-linux.sh index 1441a7ede..8dbf57fdc 100644 --- a/.pipelines/build-linux.sh +++ b/.pipelines/build-linux.sh @@ -24,3 +24,4 @@ echo "------------ Bundle Shell Extension Scripts & HELM chart ----------------- cd $DIR/../deployment/arc-k8s-extension/ServiceGroupRoot/Scripts tar -czvf ../artifacts.tar.gz ../../../../charts/azuremonitor-containers/ pushChartToAcr.sh + diff --git a/.pipelines/pipeline.user.linux.official.all_tag.all_phase.all_config.ci_prod.yml b/.pipelines/pipeline.user.linux.official.all_tag.all_phase.all_config.ci_prod.yml index ae1782d4a..9ea82d428 100644 --- a/.pipelines/pipeline.user.linux.official.all_tag.all_phase.all_config.ci_prod.yml +++ b/.pipelines/pipeline.user.linux.official.all_tag.all_phase.all_config.ci_prod.yml @@ -63,7 +63,7 @@ package: repository_name: 'cdpxlinux' # only supported ones are cdpx acr repos tag: 'ciprod' # OPTIONAL: Defaults to latest. The tag for the built image. Final tag will be 1.0.0alpha, 1.0.0-timestamp-commitID. latest: false # OPTIONAL: Defaults to false. If tag is not set to latest and this flag is set, then tag as latest as well and push latest as well. - publish_unique_tag: true # If set, the image in the registry is tagged with the unique tag generated by CDPx + publish_unique_tag: true # If set, the image in the registry is tagged with the unique tag generated by CDPx metadata_file: artifact_path: 'linux-image-meta.json' # If defined, the drop outputs relative path to the file into which JSON metadata about the created image is emitted. export_to_artifact_path: 'agentimage.tar.gz' # path for exported image and use this instead of fixed tag diff --git a/.pipelines/pipeline.user.linux.yml b/.pipelines/pipeline.user.linux.yml index 48bdcbca6..80b1b0687 100644 --- a/.pipelines/pipeline.user.linux.yml +++ b/.pipelines/pipeline.user.linux.yml @@ -63,7 +63,7 @@ package: repository_name: 'cdpxlinux' # only supported ones are cdpx acr repos tag: 'cidev' # OPTIONAL: Defaults to latest. The tag for the built image. Final tag will be 1.0.0alpha, 1.0.0-timestamp-commitID. latest: false # OPTIONAL: Defaults to false. If tag is not set to latest and this flag is set, then tag as latest as well and push latest as well. - publish_unique_tag: true # If set, the image in the registry is tagged with the unique tag generated by CDPx + publish_unique_tag: true # If set, the image in the registry is tagged with the unique tag generated by CDPx metadata_file: artifact_path: 'linux-image-meta.json' # If defined, the drop outputs relative path to the file into which JSON metadata about the created image is emitted. export_to_artifact_path: 'agentimage.tar.gz' # path for exported image and use this instead of fixed tag diff --git a/.pipelines/pipeline.user.windows.official.all_tag.all_phase.all_config.ci_prod.yml b/.pipelines/pipeline.user.windows.official.all_tag.all_phase.all_config.ci_prod.yml index d31def95c..1caf60b7b 100644 --- a/.pipelines/pipeline.user.windows.official.all_tag.all_phase.all_config.ci_prod.yml +++ b/.pipelines/pipeline.user.windows.official.all_tag.all_phase.all_config.ci_prod.yml @@ -53,7 +53,7 @@ package: repository_name: 'cdpxwin1809' # only supported ones are cdpx acr repos tag: 'win-ciprod' # OPTIONAL: Defaults to latest. The tag for the built image. Final tag will be 1.0.0alpha, 1.0.0-timestamp-commitID. latest: false # OPTIONAL: Defaults to false. If tag is not set to latest and this flag is set, then tag as latest as well and push latest as well. - publish_unique_tag: true # If set, the image in the registry is tagged with the unique tag generated by CDPx + publish_unique_tag: true # If set, the image in the registry is tagged with the unique tag generated by CDPx metadata_file: artifact_path: 'windows-image-meta.json' # If defined, the drop outputs relative path to the file into which JSON metadata about the created image is emitted. export_to_artifact_path: 'agentimage.tar.zip' # path for exported image and use this instead of fixed tag diff --git a/README.md b/README.md index ac47bf121..f0fa40e53 100644 --- a/README.md +++ b/README.md @@ -151,6 +151,25 @@ bash build-and-publish-docker-image.sh --image /: ``` > Note: format of the imagetag will be `ci`. possible values for release are test, dev, preview, dogfood, prod etc. +You can also build and push images for multiple architectures. This is powered by docker buildx +``` +cd ~/Docker-Provider/kubernetes/linux/dockerbuild +sudo docker login # if you want to publish the image to acr then login to acr via `docker login ` +# build and publish using docker buildx +bash build-and-publish-docker-image.sh --image /: --multiarch +``` + +or directly use the docker buildx commands +``` +# multiple platforms +cd ~/Docker-Provider +docker buildx build --platform linux/arm64/v8,linux/amd64 -t /: --build-arg IMAGE_TAG= -f kubernetes/linux/Dockerfile.multiarch --push . + +# single platform +cd ~/Docker-Provider +docker buildx build --platform linux/amd64 -t /: --build-arg IMAGE_TAG= -f kubernetes/linux/Dockerfile.multiarch --push . +``` + If you prefer to build docker provider shell bundle and image separately, then you can follow below instructions ##### Build Docker Provider shell bundle @@ -211,7 +230,7 @@ powershell -ExecutionPolicy bypass # switch to powershell if you are not on pow ``` ##### Developer Build optimizations -If you do not want to build the image from scratch every time you make changes during development,you can choose to build the docker images that are separated out by +If you do not want to build the image from scratch every time you make changes during development,you can choose to build the docker images that are separated out by * Base image and dependencies including agent bootstrap(setup.ps1) * Agent conf and plugin changes @@ -229,7 +248,7 @@ And then run the script to build the image consisting of code and conf changes. .\build-and-publish-dev-docker-image.ps1 -image /: # trigger build code and image and publish docker hub or acr ``` -For the subsequent builds, you can just run - +For the subsequent builds, you can just run - ``` .\build-and-publish-dev-docker-image.ps1 -image /: # trigger build code and image and publish docker hub or acr @@ -302,13 +321,13 @@ For DEV and PROD branches, automatically deployed latest yaml with latest agent ## For executing tests 1. Deploy the omsagent.yaml with your agent image. In the yaml, make sure `ISTEST` environment variable set to `true` if its not set already -2. Update the Service Principal CLIENT_ID, CLIENT_SECRET and TENANT_ID placeholder values and apply e2e-tests.yaml to execute the tests +2. Update the Service Principal CLIENT_ID, CLIENT_SECRET and TENANT_ID placeholder values and apply e2e-tests.yaml to execute the tests > Note: Service Principal requires reader role on log analytics workspace and cluster resource to query LA and metrics ``` - cd ~/Docker-Provider/test/e2e # based on your repo path - kubectl apply -f e2e-tests.yaml # this will trigger job to run the tests in sonobuoy namespace - kubectl get po -n sonobuoy # to check the pods and jobs associated to tests - ``` + cd ~/Docker-Provider/test/e2e # based on your repo path + kubectl apply -f e2e-tests.yaml # this will trigger job to run the tests in sonobuoy namespace + kubectl get po -n sonobuoy # to check the pods and jobs associated to tests + ``` 3. Download (sonobuoy)[https://github.com/vmware-tanzu/sonobuoy/releases] on your dev box to view the results of the tests ``` results=$(sonobuoy retrieve) # downloads tar file which has logs and test results @@ -319,9 +338,9 @@ For DEV and PROD branches, automatically deployed latest yaml with latest agent ## For adding new tests 1. Add the test python file with your test code under `tests` directory -2. Build the docker image, recommended to use ACR & MCR +2. Build the docker image, recommended to use ACR & MCR ``` - cd ~/Docker-Provider/test/e2e/src # based on your repo path + cd ~/Docker-Provider/test/e2e/src # based on your repo path docker login -u -p # login to acr docker build -f ./core/Dockerfile -t /: . docker push /: diff --git a/ReleaseNotes.md b/ReleaseNotes.md index 9e156c339..d0e5bad22 100644 --- a/ReleaseNotes.md +++ b/ReleaseNotes.md @@ -11,16 +11,28 @@ additional questions or comments. Note : The agent version(s) below has dates (ciprod), which indicate the agent build dates (not release dates) -### 3/11/2022 - -##### Version microsoft/oms:ciprod03112022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03112022 (linux) -##### Version microsoft/oms:win-ciprod03112022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod03112022 (windows) +### 3/17/2022 - +##### Version microsoft/oms:ciprod03172022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03172022 (linux) +##### Version microsoft/oms:win-ciprod03172022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod03172022 (windows) ##### Code change log - Linux Agent + - Multi-Arch Image to support both AMD64 and ARM64 + - Ruby upgraded to version 2.7 from 2.6 + - Fix Telegraf Permissions + - Fix ADX bug with database name - Vulnerability fixes + - MDSD updated to 1.17.0 + - HTTP Proxy support + - Retries for Log Analytics Ingestion + - ARM64 support + - Memory leak fixes for network failure scenario - Windows Agent - Bug fix for FluentBit stdout and stderr log filtering - Common - - Upgrade Go lang version from 1.14.1 to 1.15.14 + - Upgrade Go lang version from 1.14.1 to 1.15.14 + - MSI onboarding ARM template update + - AKS HTTP Proxy support + - Go packages upgrade to address vulnerabilities ### 1/31/2022 - ##### Version microsoft/oms:ciprod01312022 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod01312022 (linux) @@ -100,6 +112,7 @@ Note : The agent version(s) below has dates (ciprod), which indicate t - Fix for https://github.com/Azure/AKS/issues/2457 - In replicaset, tailing of the mdsd.err log file to agent telemetry + ### 07/13/2021 - ##### Version microsoft/oms:win-ciprod06112021-2 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod06112021-2 (windows) ##### Code change log diff --git a/build/common/installer/scripts/td-agent-bit-conf-customizer.rb b/build/common/installer/scripts/td-agent-bit-conf-customizer.rb index 1d76007a4..995d72b87 100644 --- a/build/common/installer/scripts/td-agent-bit-conf-customizer.rb +++ b/build/common/installer/scripts/td-agent-bit-conf-customizer.rb @@ -25,14 +25,14 @@ def substituteFluentBitPlaceHolders bufferMaxSize = ENV["FBIT_TAIL_BUFFER_MAX_SIZE"] memBufLimit = ENV["FBIT_TAIL_MEM_BUF_LIMIT"] - serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0) ? interval : @default_service_interval + serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0 ) ? interval : @default_service_interval serviceIntervalSetting = "Flush " + serviceInterval tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize) && bufferChunkSize.to_i > 0) ? bufferChunkSize : nil tailBufferMaxSize = (!bufferMaxSize.nil? && is_number?(bufferMaxSize) && bufferMaxSize.to_i > 0) ? bufferMaxSize : nil - if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i)) + if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) || (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i)) puts "config:warn buffer max size must be greater or equal to chunk size" tailBufferMaxSize = tailBufferChunkSize end diff --git a/build/common/installer/scripts/tomlparser.rb b/build/common/installer/scripts/tomlparser.rb index 4317f0c54..6a2f3c6d6 100644 --- a/build/common/installer/scripts/tomlparser.rb +++ b/build/common/installer/scripts/tomlparser.rb @@ -144,8 +144,8 @@ def populateSettingValuesFromConfigMap(parsedConfig) ConfigParseErrorLogger.logError("Exception while reading config map settings for cluster level container log enrichment - #{errorStr}, using defaults, please check config map for errors") end - #Get container log schema version setting - begin + #Get container log schema version setting + begin if !parsedConfig[:log_collection_settings][:schema].nil? && !parsedConfig[:log_collection_settings][:schema][:containerlog_schema_version].nil? @containerLogSchemaVersion = parsedConfig[:log_collection_settings][:schema][:containerlog_schema_version] puts "config::Using config map setting for container log schema version" @@ -168,11 +168,11 @@ def populateSettingValuesFromConfigMap(parsedConfig) begin if !parsedConfig[:log_collection_settings][:route_container_logs].nil? && !parsedConfig[:log_collection_settings][:route_container_logs][:version].nil? if !parsedConfig[:log_collection_settings][:route_container_logs][:version].empty? - @containerLogsRoute = parsedConfig[:log_collection_settings][:route_container_logs][:version] - puts "config::Using config map setting for container logs route: #{@containerLogsRoute}" - else - puts "config::Ignoring config map settings and using default value since provided container logs route value is empty" - end + @containerLogsRoute = parsedConfig[:log_collection_settings][:route_container_logs][:version] + puts "config::Using config map setting for container logs route: #{@containerLogsRoute}" + else + puts "config::Ignoring config map settings and using default value since provided container logs route value is empty" + end end rescue => errorStr ConfigParseErrorLogger.logError("Exception while reading config map settings for container logs route - #{errorStr}, using defaults, please check config map for errors") @@ -182,12 +182,12 @@ def populateSettingValuesFromConfigMap(parsedConfig) begin if !parsedConfig[:log_collection_settings][:adx_database].nil? && !parsedConfig[:log_collection_settings][:adx_database][:name].nil? if !parsedConfig[:log_collection_settings][:adx_database][:name].empty? - @adxDatabaseName = parsedConfig[:log_collection_settings][:adx_database][:name] - puts "config::Using config map setting for ADX database name : #{@adxDatabaseName}" - else - puts "config::Ignoring config map settings and using default value '#{@adxDatabaseName}' since provided adx database name value is empty" - end - else + @adxDatabaseName = parsedConfig[:log_collection_settings][:adx_database][:name] + puts "config::Using config map setting for ADX database name : #{@adxDatabaseName}" + else + puts "config::Ignoring config map settings and using default value '#{@adxDatabaseName}' since provided adx database name value is empty" + end + else puts "config::No ADX database name set, using default value : #{@adxDatabaseName}" end rescue => errorStr @@ -246,6 +246,7 @@ def populateSettingValuesFromConfigMap(parsedConfig) puts "****************End Config Processing********************" end + =begin This section generates the file that will set the environment variables for windows. This script will be called by the main.ps1 script which is the ENTRYPOINT script for the windows aks log container @@ -272,29 +273,29 @@ def get_command_windows(env_variable_name, env_variable_value) end commands = get_command_windows("AZMON_COLLECT_STDOUT_LOGS", @collectStdoutLogs) file.write(commands) - commands = get_command_windows("AZMON_LOG_TAIL_PATH", @logTailPath) + commands = get_command_windows('AZMON_LOG_TAIL_PATH', @logTailPath) file.write(commands) - commands = get_command_windows("AZMON_LOG_EXCLUSION_REGEX_PATTERN", @logExclusionRegexPattern) + commands = get_command_windows('AZMON_LOG_EXCLUSION_REGEX_PATTERN', @logExclusionRegexPattern) file.write(commands) - commands = get_command_windows("AZMON_STDOUT_EXCLUDED_NAMESPACES", @stdoutExcludeNamespaces) + commands = get_command_windows('AZMON_STDOUT_EXCLUDED_NAMESPACES', @stdoutExcludeNamespaces) file.write(commands) - commands = get_command_windows("AZMON_COLLECT_STDERR_LOGS", @collectStderrLogs) + commands = get_command_windows('AZMON_COLLECT_STDERR_LOGS', @collectStderrLogs) file.write(commands) - commands = get_command_windows("AZMON_STDERR_EXCLUDED_NAMESPACES", @stderrExcludeNamespaces) + commands = get_command_windows('AZMON_STDERR_EXCLUDED_NAMESPACES', @stderrExcludeNamespaces) file.write(commands) - commands = get_command_windows("AZMON_CLUSTER_COLLECT_ENV_VAR", @collectClusterEnvVariables) + commands = get_command_windows('AZMON_CLUSTER_COLLECT_ENV_VAR', @collectClusterEnvVariables) file.write(commands) - commands = get_command_windows("AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH", @excludePath) + commands = get_command_windows('AZMON_CLUSTER_LOG_TAIL_EXCLUDE_PATH', @excludePath) file.write(commands) - commands = get_command_windows("AZMON_CLUSTER_CONTAINER_LOG_ENRICH", @enrichContainerLogs) + commands = get_command_windows('AZMON_CLUSTER_CONTAINER_LOG_ENRICH', @enrichContainerLogs) file.write(commands) - commands = get_command_windows("AZMON_CLUSTER_COLLECT_ALL_KUBE_EVENTS", @collectAllKubeEvents) + commands = get_command_windows('AZMON_CLUSTER_COLLECT_ALL_KUBE_EVENTS', @collectAllKubeEvents) file.write(commands) - commands = get_command_windows("AZMON_CONTAINER_LOGS_ROUTE", @containerLogsRoute) + commands = get_command_windows('AZMON_CONTAINER_LOGS_ROUTE', @containerLogsRoute) file.write(commands) - commands = get_command_windows("AZMON_CONTAINER_LOG_SCHEMA_VERSION", @containerLogSchemaVersion) + commands = get_command_windows('AZMON_CONTAINER_LOG_SCHEMA_VERSION', @containerLogSchemaVersion) file.write(commands) - commands = get_command_windows("AZMON_ADX_DATABASE_NAME", @adxDatabaseName) + commands = get_command_windows('AZMON_ADX_DATABASE_NAME', @adxDatabaseName) file.write(commands) # Close file after writing all environment variables @@ -305,4 +306,4 @@ def get_command_windows(env_variable_name, env_variable_value) puts "Exception while opening file for writing config environment variables for WINDOWS LOG" puts "****************End Config Processing********************" end -end +end \ No newline at end of file diff --git a/build/linux/Makefile b/build/linux/Makefile index 3f35e1204..9203f1d73 100644 --- a/build/linux/Makefile +++ b/build/linux/Makefile @@ -11,10 +11,16 @@ BUILD_TYPE=Release PF=Linux PF_MAJOR=1 PF_MINOR=0 -PF_ARCH=x64 PF_WIDTH=64 PF_DISTRO=ULINUX -BUILD_CONFIGURATION=Linux_ULINUX_1.0_x64_64_Release + +ifeq ($(arch),) + PF_ARCH=amd64 +else + PF_ARCH=$(arch) +endif + +BUILD_CONFIGURATION=Linux_ULINUX_1.0_$(PF_ARCH)_64_Release # RM - Remove a file # RMDIR - Remove a directory @@ -90,16 +96,7 @@ endif STAGING_DIR := $(INTERMEDIATE_DIR)/staging -# For consistency, the architecture should be i686 (for x86) and x86_64 (for x64) -DOCKER_ARCH := $(shell echo $(PF_ARCH) | sed -e 's/x86$$/i686/' -e 's/x64$$/x86_64/') -OUTPUT_PACKAGE_PREFIX=docker-cimprov-$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH)-$(CONTAINER_BUILDVERSION_BUILDNR).universal.$(DOCKER_ARCH) - - -ifeq ("$(wildcard /usr/bin/dpkg-deb)","") - DPKG_LOCATION="--DPKG_LOCATION=$(BASE_DIR)/build/linux/installer/InstallBuilder/tools/bin/dpkg-deb-$(PF_ARCH)" -else - DPKG_LOCATION= -endif +OUTPUT_PACKAGE_PREFIX=docker-cimprov-$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH)-$(CONTAINER_BUILDVERSION_BUILDNR).universal.$(PF_ARCH) #-------------------------------------------------------------------------------- @@ -130,7 +127,7 @@ KIT_STATUS: fluentbitplugin : @echo "========================= Building fluentbit out_oms go plugin for logs" $(MKPATH) $(INTERMEDIATE_DIR) - make -C $(GO_SOURCE_DIR) fbplugin + make -C $(GO_SOURCE_DIR) fbplugin arch=$(PF_ARCH) $(COPY) $(GO_SOURCE_DIR)/out_oms.so $(INTERMEDIATE_DIR) rubypluginstests : @@ -192,7 +189,6 @@ kit : fluentbitplugin --VERSION=$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH) \ --RELEASE=$(CONTAINER_BUILDVERSION_BUILDNR) \ --CONTAINER_BUILD_LIBRARY=$(CONTAINERLIB_FILENAME) \ - $(DPKG_LOCATION) \ --OUTPUTFILE=$(OUTPUT_PACKAGE_PREFIX) \ --DATAFILE_PATH=$(BASE_DIR)/build/linux/installer/datafiles \ base_container.data linux.data linux_dpkg.data diff --git a/build/windows/installer/conf/fluent-bit.conf b/build/windows/installer/conf/fluent-bit.conf index f4ebfa201..1e2d8a93e 100644 --- a/build/windows/installer/conf/fluent-bit.conf +++ b/build/windows/installer/conf/fluent-bit.conf @@ -50,15 +50,6 @@ Buffer_Size 64 Mem_Buf_Limit 5m -[INPUT] - Name tcp - Tag oms.container.perf.telegraf.* - Listen 0.0.0.0 - Port 25229 - Chunk_Size 32 - Buffer_Size 64 - Mem_Buf_Limit 5m - [FILTER] Name grep Match oms.container.log.la.* diff --git a/build/windows/installer/conf/fluent.conf b/build/windows/installer/conf/fluent.conf index 54a1c9cea..a78ac58fa 100644 --- a/build/windows/installer/conf/fluent.conf +++ b/build/windows/installer/conf/fluent.conf @@ -31,7 +31,7 @@ retry_max_times 10 retry_wait 5s retry_max_interval 5m - flush_thread_count 5 + flush_thread_count 5 retry_mdm_post_wait_minutes 30 diff --git a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml index 153395727..3b48c26c4 100644 --- a/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-daemonset.yaml @@ -113,6 +113,12 @@ spec: - mountPath: /etc/omsagent-secret name: omsagent-secret readOnly: true + {{- if and (.Values.Azure.proxySettings.isProxyEnabled) (.Values.Azure.proxySettings.proxyCert) }} + - mountPath: /etc/ssl/certs/proxy-cert.crt + subPath: PROXYCERT.crt + name: omsagent-secret + readOnly: true + {{- end }} - mountPath: /etc/config/settings name: settings-vol-config readOnly: true @@ -179,6 +185,12 @@ spec: - mountPath: /etc/omsagent-secret name: omsagent-secret readOnly: true + {{- if and (.Values.Azure.proxySettings.isProxyEnabled) (.Values.Azure.proxySettings.proxyCert) }} + - mountPath: /etc/ssl/certs/proxy-cert.crt + subPath: PROXYCERT.crt + name: omsagent-secret + readOnly: true + {{- end }} - mountPath: /etc/config/settings name: settings-vol-config readOnly: true diff --git a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml index a7ea8b097..55b1f4a8d 100644 --- a/charts/azuremonitor-containers/templates/omsagent-deployment.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-deployment.yaml @@ -99,6 +99,12 @@ spec: - mountPath: /etc/omsagent-secret name: omsagent-secret readOnly: true + {{- if and (.Values.Azure.proxySettings.isProxyEnabled) (.Values.Azure.proxySettings.proxyCert) }} + - mountPath: /etc/ssl/certs/proxy-cert.crt + subPath: PROXYCERT.crt + name: omsagent-secret + readOnly: true + {{- end }} - mountPath : /etc/config name: omsagent-rs-config - mountPath: /etc/config/settings diff --git a/charts/azuremonitor-containers/templates/omsagent-secret.yaml b/charts/azuremonitor-containers/templates/omsagent-secret.yaml index 8c245338c..03d9a48c5 100644 --- a/charts/azuremonitor-containers/templates/omsagent-secret.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-secret.yaml @@ -13,19 +13,14 @@ data: WSID: {{ required "A valid workspace id is required!" .Values.omsagent.secret.wsid | b64enc | quote }} KEY: {{ required "A valid workspace key is required!" .Values.omsagent.secret.key | b64enc | quote }} DOMAIN: {{ .Values.omsagent.domain | b64enc | quote }} - {{- $httpsProxyDict := urlParse .Values.Azure.proxySettings.httpsProxy -}} - {{- $httpProxyDict := urlParse .Values.Azure.proxySettings.httpProxy -}} - {{- if and (and (.Values.Azure.proxySettings.isProxyEnabled) (.Values.Azure.proxySettings.httpsProxy)) ($httpsProxyDict.userinfo) }} + {{- if and (.Values.Azure.proxySettings.isProxyEnabled) (.Values.Azure.proxySettings.httpsProxy) }} PROXY: {{ .Values.Azure.proxySettings.httpsProxy | b64enc | quote }} - {{- else if and (and (.Values.Azure.proxySettings.isProxyEnabled) (.Values.Azure.proxySettings.httpsProxy)) (empty $httpsProxyDict.userinfo) }} - # adding arbitrary creds since omsagent expects arbitrary creds in case of no auth - PROXY: {{ urlJoin (dict "scheme" $httpsProxyDict.scheme "userinfo" "admin:secret" "host" $httpsProxyDict.host) | b64enc | quote }} - {{- else if and (and (.Values.Azure.proxySettings.isProxyEnabled) (.Values.Azure.proxySettings.httpProxy)) ($httpProxyDict.userinfo) }} + {{- else if and (.Values.Azure.proxySettings.isProxyEnabled) (.Values.Azure.proxySettings.httpProxy) }} PROXY: {{ .Values.Azure.proxySettings.httpProxy | b64enc | quote }} - {{- else if and (and (.Values.Azure.proxySettings.isProxyEnabled) (.Values.Azure.proxySettings.httpProxy)) (empty $httpProxyDict.userinfo) }} - # adding arbitrary creds since omsagent expects arbitrary creds in case of no auth - PROXY: {{ urlJoin (dict "scheme" $httpProxyDict.scheme "userinfo" "admin:secret" "host" $httpProxyDict.host) | b64enc | quote }} {{- else if ne .Values.omsagent.proxy "" }} PROXY: {{ .Values.omsagent.proxy | b64enc | quote }} {{- end }} + {{- if and (.Values.Azure.proxySettings.isProxyEnabled) (.Values.Azure.proxySettings.proxyCert) }} + PROXYCERT.crt: {{.Values.Azure.proxySettings.proxyCert | b64enc | quote}} + {{- end }} {{- end }} diff --git a/charts/azuremonitor-containers/values.yaml b/charts/azuremonitor-containers/values.yaml index 3ed5d780f..4460f7756 100644 --- a/charts/azuremonitor-containers/values.yaml +++ b/charts/azuremonitor-containers/values.yaml @@ -21,11 +21,11 @@ Azure: omsagent: image: repo: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod" - tag: "ciprod03112022" - tagWindows: "win-ciprod03112022" + tag: "ciprod03172022" + tagWindows: "win-ciprod03172022" pullPolicy: IfNotPresent dockerProviderVersion: "16.0.0-0" - agentVersion: "azure-mdsd-1.14.2" + agentVersion: "azure-mdsd-1.17.0" winAgentVersion: "0.0.0-0" # there is no base agent version for windows agent # The priority used by the omsagent priority class for the daemonset pods diff --git a/deployment/arc-k8s-extension/ServiceGroupRoot/Parameters/ContainerInsightsExtension.Parameters.json b/deployment/arc-k8s-extension/ServiceGroupRoot/Parameters/ContainerInsightsExtension.Parameters.json index c38c67e00..69e1bcf35 100644 --- a/deployment/arc-k8s-extension/ServiceGroupRoot/Parameters/ContainerInsightsExtension.Parameters.json +++ b/deployment/arc-k8s-extension/ServiceGroupRoot/Parameters/ContainerInsightsExtension.Parameters.json @@ -39,7 +39,7 @@ "name": "CHART_VERSION", "value": "__CHART_VERSION__" } - ], + ], "identity": { "type": "userAssigned", "userAssignedIdentities": [ @@ -49,4 +49,4 @@ } } ] -} +} \ No newline at end of file diff --git a/deployment/arc-k8s-extension/ServiceGroupRoot/RolloutSpecs/Public.Canary.RolloutSpec.json b/deployment/arc-k8s-extension/ServiceGroupRoot/RolloutSpecs/Public.Canary.RolloutSpec.json index ea396bbe4..2d0149e24 100644 --- a/deployment/arc-k8s-extension/ServiceGroupRoot/RolloutSpecs/Public.Canary.RolloutSpec.json +++ b/deployment/arc-k8s-extension/ServiceGroupRoot/RolloutSpecs/Public.Canary.RolloutSpec.json @@ -2,8 +2,8 @@ "$schema": "http://schema.express.azure.com/schemas/2015-01-01-alpha/RolloutSpec.json", "ContentVersion": "1.0.0.0", "RolloutMetadata": { - "ServiceModelPath": "ServiceModels//Public.ServiceModel.json", - "ScopeBindingsPath": "ScopeBindings//Public.ScopeBindings.json", + "ServiceModelPath": "ServiceModels//Public.ServiceModel.json", + "ScopeBindingsPath": "ScopeBindings//Public.ScopeBindings.json", "Name": "ContainerInsightsExtension-Canary", "RolloutType": "Major", "BuildSource": { @@ -15,7 +15,7 @@ "email": { "to": "omscontainers@microsoft.com" } - } + } }, "orchestratedSteps": [ { @@ -26,4 +26,4 @@ "dependsOn": [ ] } ] -} +} \ No newline at end of file diff --git a/deployment/arc-k8s-extension/ServiceGroupRoot/ScopeBindings/Public.ScopeBindings.json b/deployment/arc-k8s-extension/ServiceGroupRoot/ScopeBindings/Public.ScopeBindings.json index 97f103efa..bf61ab7fd 100644 --- a/deployment/arc-k8s-extension/ServiceGroupRoot/ScopeBindings/Public.ScopeBindings.json +++ b/deployment/arc-k8s-extension/ServiceGroupRoot/ScopeBindings/Public.ScopeBindings.json @@ -1,18 +1,18 @@ { "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/scopeBindings.json", "contentVersion": "0.0.0.1", - "scopeBindings": [ + "scopeBindings": [ { "scopeTagName": "Canary", "bindings": [ { "find": "__RELEASE_STAGE__", "replaceWith": "Canary" - }, + }, { "find": "__ACR_NAME__", "replaceWith": "$(ACRName)" - }, + }, { "find": "__CHART_VERSION__", "replaceWith": "$(ChartVersion)" @@ -20,7 +20,7 @@ { "find": "__MANAGED_IDENTITY__", "replaceWith": "$(ManagedIdentity)" - } + } ] }, { @@ -29,11 +29,11 @@ { "find": "__RELEASE_STAGE__", "replaceWith": "Pilot" - }, + }, { "find": "__ACR_NAME__", "replaceWith": "$(ACRName)" - }, + }, { "find": "__CHART_VERSION__", "replaceWith": "$(ChartVersion)" @@ -41,7 +41,7 @@ { "find": "__MANAGED_IDENTITY__", "replaceWith": "$(ManagedIdentity)" - } + } ] }, { @@ -50,11 +50,11 @@ { "find": "__RELEASE_STAGE__", "replaceWith": "MediumLow" - }, + }, { "find": "__ACR_NAME__", "replaceWith": "$(ACRName)" - }, + }, { "find": "__CHART_VERSION__", "replaceWith": "$(ChartVersion)" @@ -62,7 +62,7 @@ { "find": "__MANAGED_IDENTITY__", "replaceWith": "$(ManagedIdentity)" - } + } ] }, { @@ -71,11 +71,11 @@ { "find": "__RELEASE_STAGE__", "replaceWith": "MediumHigh" - }, + }, { "find": "__ACR_NAME__", "replaceWith": "$(ACRName)" - }, + }, { "find": "__CHART_VERSION__", "replaceWith": "$(ChartVersion)" @@ -83,7 +83,7 @@ { "find": "__MANAGED_IDENTITY__", "replaceWith": "$(ManagedIdentity)" - } + } ] }, { @@ -92,11 +92,11 @@ { "find": "__RELEASE_STAGE__", "replaceWith": "HighLoad" - }, + }, { "find": "__ACR_NAME__", "replaceWith": "$(ACRName)" - }, + }, { "find": "__CHART_VERSION__", "replaceWith": "$(ChartVersion)" @@ -104,7 +104,7 @@ { "find": "__MANAGED_IDENTITY__", "replaceWith": "$(ManagedIdentity)" - } + } ] }, { @@ -113,11 +113,11 @@ { "find": "__RELEASE_STAGE__", "replaceWith": "FF" - }, + }, { "find": "__ACR_NAME__", "replaceWith": "$(ACRName)" - }, + }, { "find": "__CHART_VERSION__", "replaceWith": "$(ChartVersion)" @@ -125,7 +125,7 @@ { "find": "__MANAGED_IDENTITY__", "replaceWith": "$(ManagedIdentity)" - } + } ] }, { @@ -134,11 +134,11 @@ { "find": "__RELEASE_STAGE__", "replaceWith": "MC" - }, + }, { "find": "__ACR_NAME__", "replaceWith": "$(ACRName)" - }, + }, { "find": "__CHART_VERSION__", "replaceWith": "$(ChartVersion)" @@ -146,8 +146,8 @@ { "find": "__MANAGED_IDENTITY__", "replaceWith": "$(ManagedIdentity)" - } + } ] - } + } ] -} +} \ No newline at end of file diff --git a/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts/pushChartToAcr.sh b/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts/pushChartToAcr.sh index 18d5eef5a..99421b122 100644 --- a/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts/pushChartToAcr.sh +++ b/deployment/arc-k8s-extension/ServiceGroupRoot/Scripts/pushChartToAcr.sh @@ -10,7 +10,7 @@ export REPO_TYPE="stable" export CANARY_REGION_REPO_PATH="azuremonitor/containerinsights/canary/${REPO_TYPE}/azuremonitor-containers" # pilot region export PILOT_REGION_REPO_PATH="azuremonitor/containerinsights/prod1/${REPO_TYPE}/azuremonitor-containers" -# light load regions +# light load regions export LIGHT_LOAD_REGION_REPO_PATH="azuremonitor/containerinsights/prod2/${REPO_TYPE}/azuremonitor-containers" # medium load regions export MEDIUM_LOAD_REGION_REPO_PATH="azuremonitor/containerinsights/prod3/${REPO_TYPE}/azuremonitor-containers" @@ -18,7 +18,7 @@ export MEDIUM_LOAD_REGION_REPO_PATH="azuremonitor/containerinsights/prod3/${REPO export HIGH_LOAD_REGION_REPO_PATH="azuremonitor/containerinsights/prod4/${REPO_TYPE}/azuremonitor-containers" # FairFax regions export FF_REGION_REPO_PATH="azuremonitor/containerinsights/prod5/${REPO_TYPE}/azuremonitor-containers" -# Mooncake regions +# Mooncake regions export MC_REGION_REPO_PATH="azuremonitor/containerinsights/prod6/${REPO_TYPE}/azuremonitor-containers" # pull chart from previous stage mcr and push chart to next stage acr @@ -35,6 +35,7 @@ pull_chart_from_source_mcr_to_push_to_dest_acr() { echo "-e error dest acr path must be provided " exit 1 fi + echo "Pulling chart from MCR:${srcMcrFullPath} ..." helm chart pull ${srcMcrFullPath} if [ $? -eq 0 ]; then @@ -42,34 +43,34 @@ pull_chart_from_source_mcr_to_push_to_dest_acr() { else echo "-e error Pulling chart from MCR:${srcMcrFullPath} failed. Please review Ev2 pipeline logs for more details on the error." exit 1 - fi + fi - echo "Exporting chart to current directory ..." + echo "Exporting chart to current directory ..." helm chart export ${srcMcrFullPath} if [ $? -eq 0 ]; then echo "Exporting chart to current directory completed successfully." else echo "-e error Exporting chart to current directory failed. Please review Ev2 pipeline logs for more details on the error." exit 1 - fi + fi - echo "save the chart locally with dest acr full path : ${destAcrFullPath} ..." - helm chart save azuremonitor-containers/ ${destAcrFullPath} - if [ $? -eq 0 ]; then + echo "save the chart locally with dest acr full path : ${destAcrFullPath} ..." + helm chart save azuremonitor-containers/ ${destAcrFullPath} + if [ $? -eq 0 ]; then echo "save the chart locally with dest acr full path : ${destAcrFullPath} completed successfully." - else + else echo "-e error save the chart locally with dest acr full path : ${destAcrFullPath} failed. Please review Ev2 pipeline logs for more details on the error." exit 1 - fi - + fi + echo "pushing the chart to acr path: ${destAcrFullPath} ..." - helm chart push ${destAcrFullPath} - if [ $? -eq 0 ]; then + helm chart push ${destAcrFullPath} + if [ $? -eq 0 ]; then echo "pushing the chart to acr path: ${destAcrFullPath} completed successfully." - else + else echo "-e error pushing the chart to acr path: ${destAcrFullPath} failed. Please review Ev2 pipeline logs for more details on the error." exit 1 - fi + fi } # push to local release candidate chart to canary region @@ -80,23 +81,23 @@ push_local_chart_to_canary_region() { exit 1 fi - echo "save the chart locally with dest acr full path : ${destAcrFullPath} ..." + echo "save the chart locally with dest acr full path : ${destAcrFullPath} ..." helm chart save charts/azuremonitor-containers/ $destAcrFullPath - if [ $? -eq 0 ]; then + if [ $? -eq 0 ]; then echo "save the chart locally with dest acr full path : ${destAcrFullPath} completed." - else + else echo "-e error save the chart locally with dest acr full path : ${destAcrFullPath} failed. Please review Ev2 pipeline logs for more details on the error." exit 1 - fi + fi echo "pushing the chart to acr path: ${destAcrFullPath} ..." helm chart push $destAcrFullPath - if [ $? -eq 0 ]; then + if [ $? -eq 0 ]; then echo "pushing the chart to acr path: ${destAcrFullPath} completed successfully." - else + else echo "-e error pushing the chart to acr path: ${destAcrFullPath} failed.Please review Ev2 pipeline logs for more details on the error." exit 1 - fi + fi } echo "START - Release stage : ${RELEASE_STAGE}" @@ -116,7 +117,7 @@ else fi ACCESS_TOKEN=$(az acr login --name ${ACR_NAME} --expose-token --output tsv --query accessToken) -if [ $? -ne 0 ]; then +if [ $? -ne 0 ]; then echo "-e error az acr login failed. Please review the Ev2 pipeline logs for more details on the error." exit 1 fi @@ -128,64 +129,64 @@ if [ $? -eq 0 ]; then else echo "-e error login to acr:${ACR_NAME} using helm failed. Please review Ev2 pipeline logs for more details on the error." exit 1 -fi +fi case $RELEASE_STAGE in Canary) echo "START: Release stage - Canary" - destAcrFullPath=${ACR_NAME}/public/${CANARY_REGION_REPO_PATH}:${CHART_VERSION} - push_local_chart_to_canary_region $destAcrFullPath + destAcrFullPath=${ACR_NAME}/public/${CANARY_REGION_REPO_PATH}:${CHART_VERSION} + push_local_chart_to_canary_region $destAcrFullPath echo "END: Release stage - Canary" ;; - Pilot | Prod1) - echo "START: Release stage - Pilot" - srcMcrFullPath=${MCR_NAME}/${CANARY_REGION_REPO_PATH}:${CHART_VERSION} - destAcrFullPath=${ACR_NAME}/public/${PILOT_REGION_REPO_PATH}:${CHART_VERSION} - pull_chart_from_source_mcr_to_push_to_dest_acr $srcMcrFullPath $destAcrFullPath - echo "END: Release stage - Pilot" + Pilot | Prod1) + echo "START: Release stage - Pilot" + srcMcrFullPath=${MCR_NAME}/${CANARY_REGION_REPO_PATH}:${CHART_VERSION} + destAcrFullPath=${ACR_NAME}/public/${PILOT_REGION_REPO_PATH}:${CHART_VERSION} + pull_chart_from_source_mcr_to_push_to_dest_acr $srcMcrFullPath $destAcrFullPath + echo "END: Release stage - Pilot" ;; - LightLoad | Pord2) - echo "START: Release stage - Light Load Regions" + LightLoad | Pord2) + echo "START: Release stage - Light Load Regions" srcMcrFullPath=${MCR_NAME}/${PILOT_REGION_REPO_PATH}:${CHART_VERSION} destAcrFullPath=${ACR_NAME}/public/${LIGHT_LOAD_REGION_REPO_PATH}:${CHART_VERSION} - pull_chart_from_source_mcr_to_push_to_dest_acr $srcMcrFullPath $destAcrFullPath - echo "END: Release stage - Light Load Regions" + pull_chart_from_source_mcr_to_push_to_dest_acr $srcMcrFullPath $destAcrFullPath + echo "END: Release stage - Light Load Regions" ;; - - MediumLoad | Prod3) - echo "START: Release stage - Medium Load Regions" + + MediumLoad | Prod3) + echo "START: Release stage - Medium Load Regions" srcMcrFullPath=${MCR_NAME}/${LIGHT_LOAD_REGION_REPO_PATH}:${CHART_VERSION} destAcrFullPath=${ACR_NAME}/public/${MEDIUM_LOAD_REGION_REPO_PATH}:${CHART_VERSION} - pull_chart_from_source_mcr_to_push_to_dest_acr $srcMcrFullPath $destAcrFullPath + pull_chart_from_source_mcr_to_push_to_dest_acr $srcMcrFullPath $destAcrFullPath echo "END: Release stage - Medium Load Regions" ;; - HighLoad | Prod4) - echo "START: Release stage - High Load Regions" - srcMcrFullPath=${MCR_NAME}/${MEDIUM_LOAD_REGION_REPO_PATH}:${CHART_VERSION} - destAcrFullPath=${ACR_NAME}/public/${HIGH_LOAD_REGION_REPO_PATH}:${CHART_VERSION} - pull_chart_from_source_mcr_to_push_to_dest_acr $srcMcrFullPath $destAcrFullPath - echo "END: Release stage - High Load Regions" - ;; - - FF | Prod5) - echo "START: Release stage - FF" - srcMcrFullPath=${MCR_NAME}/${HIGH_LOAD_REGION_REPO_PATH}:${CHART_VERSION} + HighLoad | Prod4) + echo "START: Release stage - High Load Regions" + srcMcrFullPath=${MCR_NAME}/${MEDIUM_LOAD_REGION_REPO_PATH}:${CHART_VERSION} + destAcrFullPath=${ACR_NAME}/public/${HIGH_LOAD_REGION_REPO_PATH}:${CHART_VERSION} + pull_chart_from_source_mcr_to_push_to_dest_acr $srcMcrFullPath $destAcrFullPath + echo "END: Release stage - High Load Regions" + ;; + + FF | Prod5) + echo "START: Release stage - FF" + srcMcrFullPath=${MCR_NAME}/${HIGH_LOAD_REGION_REPO_PATH}:${CHART_VERSION} destAcrFullPath=${ACR_NAME}/public/${FF_REGION_REPO_PATH}:${CHART_VERSION} - pull_chart_from_source_mcr_to_push_to_dest_acr $srcMcrFullPath $destAcrFullPath - echo "END: Release stage - FF" - ;; + pull_chart_from_source_mcr_to_push_to_dest_acr $srcMcrFullPath $destAcrFullPath + echo "END: Release stage - FF" + ;; MC | Prod6) - echo "START: Release stage - MC" - srcMcrFullPath=${MCR_NAME}/${FF_REGION_REPO_PATH}:${CHART_VERSION} + echo "START: Release stage - MC" + srcMcrFullPath=${MCR_NAME}/${FF_REGION_REPO_PATH}:${CHART_VERSION} destAcrFullPath=${ACR_NAME}/public/${MC_REGION_REPO_PATH}:${CHART_VERSION} pull_chart_from_source_mcr_to_push_to_dest_acr $srcMcrFullPath $destAcrFullPath - echo "END: Release stage - MC" - ;; + echo "END: Release stage - MC" + ;; *) echo -n "unknown release stage" @@ -193,4 +194,4 @@ case $RELEASE_STAGE in ;; esac -echo "END - Release stage : ${RELEASE_STAGE}" +echo "END - Release stage : ${RELEASE_STAGE}" \ No newline at end of file diff --git a/deployment/arc-k8s-extension/ServiceGroupRoot/ServiceModels/Public.ServiceModel.json b/deployment/arc-k8s-extension/ServiceGroupRoot/ServiceModels/Public.ServiceModel.json index c53bb5aca..6f565d4c4 100644 --- a/deployment/arc-k8s-extension/ServiceGroupRoot/ServiceModels/Public.ServiceModel.json +++ b/deployment/arc-k8s-extension/ServiceGroupRoot/ServiceModels/Public.ServiceModel.json @@ -28,17 +28,17 @@ ] } ], - "ServiceResourceGroups": [ + "ServiceResourceGroups": [ { "AzureResourceGroupName": "ContainerInsightsExtension-Canary-Release", "Location": "eastus2", "InstanceOf": "ARC-Extension-ServiceResourceGroupDefinition", - "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", + "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", "ScopeTags": [ { "Name": "Canary" } - ], + ], "ServiceResources": [ { "Name": "PushChartToACR-Canary", @@ -51,12 +51,12 @@ "AzureResourceGroupName": "ContainerInsightsExtension-Pilot-Release", "Location": "eastus2", "InstanceOf": "ARC-Extension-ServiceResourceGroupDefinition", - "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", + "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", "ScopeTags": [ { "Name": "Pilot" } - ], + ], "ServiceResources": [ { "Name": "PushChartToACR-Pilot", @@ -69,12 +69,12 @@ "AzureResourceGroupName": "ContainerInsightsExtension-LightLoad-Release", "Location": "eastus2", "InstanceOf": "ARC-Extension-ServiceResourceGroupDefinition", - "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", + "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", "ScopeTags": [ { "Name": "LightLoad" } - ], + ], "ServiceResources": [ { "Name": "PushChartToACR-LightLoad", @@ -87,12 +87,12 @@ "AzureResourceGroupName": "ContainerInsightsExtension-MediumLoad-Release", "Location": "eastus2", "InstanceOf": "ARC-Extension-ServiceResourceGroupDefinition", - "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", + "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", "ScopeTags": [ { "Name": "MediumLoad" } - ], + ], "ServiceResources": [ { "Name": "PushChartToACR-MediumLoad", @@ -105,12 +105,12 @@ "AzureResourceGroupName": "ContainerInsightsExtension-HighLoad-Release", "Location": "eastus2", "InstanceOf": "ARC-Extension-ServiceResourceGroupDefinition", - "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", + "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", "ScopeTags": [ { "Name": "HighLoad" } - ], + ], "ServiceResources": [ { "Name": "PushChartToACR-HighLoad", @@ -123,12 +123,12 @@ "AzureResourceGroupName": "ContainerInsightsExtension-FF-Release", "Location": "eastus2", "InstanceOf": "ARC-Extension-ServiceResourceGroupDefinition", - "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", + "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", "ScopeTags": [ { "Name": "FF" } - ], + ], "ServiceResources": [ { "Name": "PushChartToACR-FF", @@ -141,12 +141,12 @@ "AzureResourceGroupName": "ContainerInsightsExtension-MC-Release", "Location": "eastus2", "InstanceOf": "ARC-Extension-ServiceResourceGroupDefinition", - "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", + "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", "ScopeTags": [ { "Name": "MC" } - ], + ], "ServiceResources": [ { "Name": "PushChartToACR-MC", @@ -154,6 +154,6 @@ "RolloutParametersPath": "Parameters\\ContainerInsightsExtension.Parameters.json" } ] - } + } ] - } + } \ No newline at end of file diff --git a/deployment/multiarch-agent-deployment/ServiceGroupRoot/Parameters/ContainerInsights.Linux.Parameters.json b/deployment/multiarch-agent-deployment/ServiceGroupRoot/Parameters/ContainerInsights.Linux.Parameters.json new file mode 100644 index 000000000..70d0950a2 --- /dev/null +++ b/deployment/multiarch-agent-deployment/ServiceGroupRoot/Parameters/ContainerInsights.Linux.Parameters.json @@ -0,0 +1,68 @@ +{ + "$schema": "http://schema.express.azure.com/schemas/2015-01-01-alpha/RolloutParameters.json", + "contentVersion": "1.0.0.0", + "wait": [ + { + "name": "waitSdpBakeTime", + "properties": { + "duration": "PT24H" + } + } + ], + "shellExtensions": [ + { + "name": "PushAgentToACR", + "type": "ShellExtensionType", + "properties": { + "maxexecutiontime": "PT1H" + }, + "package": { + "reference": { + "path": "artifacts.tar.gz" + } + }, + "launch": { + "command": [ + "/bin/bash", + "pushAgentToAcr.sh" + ], + "environmentVariables": [ + { + "name": "ACR_NAME", + "value": "__ACR_NAME__" + }, + { + "name": "AGENT_RELEASE", + "value": "__AGENT_RELEASE__" + }, + { + "name": "AGENT_IMAGE_TAG_SUFFIX", + "value": "__AGENT_IMAGE_TAG_SUFFIX__" + }, + { + "name": "AGENT_IMAGE_FULL_PATH", + "value": "public/azuremonitor/containerinsights/__AGENT_RELEASE__:__AGENT_RELEASE____AGENT_IMAGE_TAG_SUFFIX__" + }, + { + "name": "CDPX_REGISTRY", + "value": "__CDPX_LINUX_REGISTRY__" + }, + { + "name": "CDPX_REPO_NAME", + "value": "__CDPX_LINUX_REPO_NAME__" + }, + { + "name": "CDPX_TAG", + "value": "__CDPX_LINUX_TAG__" + } + ], + "identity": { + "type": "userAssigned", + "userAssignedIdentities": [ + "__MANAGED_IDENTITY__" + ] + } + } + } + ] + } \ No newline at end of file diff --git a/deployment/multiarch-agent-deployment/ServiceGroupRoot/RolloutSpecs/RolloutSpecs.json b/deployment/multiarch-agent-deployment/ServiceGroupRoot/RolloutSpecs/RolloutSpecs.json new file mode 100644 index 000000000..250878590 --- /dev/null +++ b/deployment/multiarch-agent-deployment/ServiceGroupRoot/RolloutSpecs/RolloutSpecs.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/rolloutSpecification.json", + "ContentVersion": "1.0.0.0", + "RolloutMetadata": { + "ServiceModelPath": "ServiceModels//Public.ServiceModel.json", + "ScopeBindingsPath": "ScopeBindings//Public.ScopeBindings.json", + "Name": "ContainerInsightsAgent", + "RolloutType": "Major", + "BuildSource": { + "Parameters": { + "VersionFile": "buildver.txt" + } + }, + "Notification": { + "Email": { + "To": "omscontainers@microsoft.com" + } + } + }, + "OrchestratedSteps": [ + { + "name": "PushLinuxAgent", + "targetType": "ServiceResource", + "targetName": "PushLinuxAgent", + "actions": [ "Shell/PushAgentToACR" ], + "dependsOn": [ ] + } + ] + } \ No newline at end of file diff --git a/deployment/multiarch-agent-deployment/ServiceGroupRoot/ScopeBindings/Public.ScopeBindings.json b/deployment/multiarch-agent-deployment/ServiceGroupRoot/ScopeBindings/Public.ScopeBindings.json new file mode 100644 index 000000000..cbc6db8b3 --- /dev/null +++ b/deployment/multiarch-agent-deployment/ServiceGroupRoot/ScopeBindings/Public.ScopeBindings.json @@ -0,0 +1,51 @@ +{ + "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/scopeBindings.json", + "contentVersion": "0.0.0.1", + "scopeBindings": [ + { + "scopeTagName": "Global", + "bindings": [ + { + "find": "__ACR_NAME__", + "replaceWith": "$(ACRName)" + }, + { + "find": "__AGENT_RELEASE__", + "replaceWith": "$(AgentRelease)" + }, + { + "find": "__AGENT_IMAGE_TAG_SUFFIX__", + "replaceWith": "$(AgentImageTagSuffix)" + }, + { + "find": "__MANAGED_IDENTITY__", + "replaceWith": "$(ManagedIdentity)" + }, + { + "find": "__CDPX_LINUX_REGISTRY__", + "replaceWith": "$(CDPXLinuxRegistry)" + }, + { + "find": "__CDPX_WINDOWS_REGISTRY__", + "replaceWith": "$(CDPXWindowsRegistry)" + }, + { + "find": "__CDPX_LINUX_TAG__", + "replaceWith": "$(CDPXLinuxTag)" + }, + { + "find": "__CDPX_WINDOWS_TAG__", + "replaceWith": "$(CDPXWindowsTag)" + }, + { + "find": "__CDPX_LINUX_REPO_NAME__", + "replaceWith": "$(CDPXLinuxRepoName)" + }, + { + "find": "__CDPX_WINDOWS_REPO_NAME__", + "replaceWith": "$(CDPXWindowsRepoName)" + } + ] + } + ] +} \ No newline at end of file diff --git a/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts/pushAgentToAcr.sh b/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts/pushAgentToAcr.sh new file mode 100644 index 000000000..d39cedde0 --- /dev/null +++ b/deployment/multiarch-agent-deployment/ServiceGroupRoot/Scripts/pushAgentToAcr.sh @@ -0,0 +1,72 @@ +#!/bin/bash +set -e + +# Note - This script used in the pipeline as inline script + +if [ -z $AGENT_IMAGE_TAG_SUFFIX ]; then + echo "-e error value of AGENT_IMAGE_TAG_SUFFIX variable shouldnt be empty. check release variables" + exit 1 +fi + +if [ -z $AGENT_RELEASE ]; then + echo "-e error AGENT_RELEASE shouldnt be empty. check release variables" + exit 1 +fi + +#Make sure that tag being pushed will not overwrite an existing tag in mcr +MCR_TAG_RESULT="`wget -qO- https://mcr.microsoft.com/v2/azuremonitor/containerinsights/ciprod/tags/list`" +if [ $? -ne 0 ]; then + echo "-e error unable to get list of mcr tags for azuremonitor/containerinsights/ciprod repository" + exit 1 +fi +TAG_EXISTS=$(echo $MCR_TAG_RESULT | jq '.tags | contains(["'"$AGENT_RELEASE$AGENT_IMAGE_TAG_SUFFIX"'"])') + +if $TAG_EXISTS; then + echo "-e error ${AGENT_IMAGE_TAG_SUFFIX} already exists in mcr. make sure the image tag is unique" + exit 1 +fi + +if [ -z $AGENT_IMAGE_FULL_PATH ]; then + echo "-e error AGENT_IMAGE_FULL_PATH shouldnt be empty. check release variables" + exit 1 +fi + +if [ -z $CDPX_TAG ]; then + echo "-e error value of CDPX_TAG shouldn't be empty. check release variables" + exit 1 +fi + +if [ -z $CDPX_REGISTRY ]; then + echo "-e error value of CDPX_REGISTRY shouldn't be empty. check release variables" + exit 1 +fi + +if [ -z $CDPX_REPO_NAME ]; then + echo "-e error value of CDPX_REPO_NAME shouldn't be empty. check release variables" + exit 1 +fi + +if [ -z $ACR_NAME ]; then + echo "-e error value of ACR_NAME shouldn't be empty. check release variables" + exit 1 +fi + + +#Login to az cli and authenticate to acr +echo "Login cli using managed identity" +az login --identity +if [ $? -eq 0 ]; then + echo "Logged in successfully" +else + echo "-e error failed to login to az with managed identity credentials" + exit 1 +fi + +echo "Pushing ${AGENT_IMAGE_FULL_PATH} to ${ACR_NAME}" +az acr import --name $ACR_NAME --registry $CDPX_REGISTRY --source official/${CDPX_REPO_NAME}:${CDPX_TAG} --image $AGENT_IMAGE_FULL_PATH +if [ $? -eq 0 ]; then + echo "Retagged and pushed image successfully" +else + echo "-e error failed to retag and push image to destination ACR" + exit 1 +fi \ No newline at end of file diff --git a/deployment/multiarch-agent-deployment/ServiceGroupRoot/ServiceModels/Public.ServiceModel.json b/deployment/multiarch-agent-deployment/ServiceGroupRoot/ServiceModels/Public.ServiceModel.json new file mode 100644 index 000000000..c3b00340a --- /dev/null +++ b/deployment/multiarch-agent-deployment/ServiceGroupRoot/ServiceModels/Public.ServiceModel.json @@ -0,0 +1,51 @@ +{ + "$schema": "https://ev2schema.azure.net/schemas/2020-01-01/serviceModel.json", + "contentVersion": "1.0.0.2", + "ServiceMetadata": { + "ServiceGroup": "ContainerInsightsAgent", + "Environment": "Prod" + }, + "ServiceResourceGroupDefinitions": [ + { + "Name": "CI-Agent-ServiceResourceGroupDefinition", + "ServiceResourceDefinitions": [ + { + "Name": "ShellExtension", + "ComposedOf": { + "Extension": { + "Shell": [ + { + "type": "ShellExtensionType", + "properties": { + "imageName": "adm-ubuntu-1804-l", + "imageVersion": "v18" + } + } + ] + } + } + } + ] + } + ], + "ServiceResourceGroups": [ + { + "AzureResourceGroupName": "ContainerInsights-MultiArch-Agent-Release", + "Location": "eastus2", + "InstanceOf": "CI-Agent-ServiceResourceGroupDefinition", + "AzureSubscriptionId": "30c56c3a-54da-46ea-b004-06eb33432687", + "ScopeTags": [ + { + "Name": "Global" + } + ], + "ServiceResources": [ + { + "Name": "PushLinuxAgent", + "InstanceOf": "ShellExtension", + "RolloutParametersPath": "Parameters\\ContainerInsights.Linux.Parameters.json" + } + ] + } + ] + } \ No newline at end of file diff --git a/deployment/multiarch-agent-deployment/ServiceGroupRoot/buildver.txt b/deployment/multiarch-agent-deployment/ServiceGroupRoot/buildver.txt new file mode 100644 index 000000000..bd2666abb --- /dev/null +++ b/deployment/multiarch-agent-deployment/ServiceGroupRoot/buildver.txt @@ -0,0 +1 @@ +1.0.0.0 \ No newline at end of file diff --git a/kubernetes/container-azm-ms-agentconfig.yaml b/kubernetes/container-azm-ms-agentconfig.yaml index 00ee4628e..328acb201 100644 --- a/kubernetes/container-azm-ms-agentconfig.yaml +++ b/kubernetes/container-azm-ms-agentconfig.yaml @@ -129,7 +129,7 @@ data: # Alertable metrics configuration settings for completed jobs count [alertable_metrics_configuration_settings.job_completion_threshold] - # Threshold for completed job count , metric will be sent only for those jobs which were completed earlier than the following threshold + # Threshold for completed job count , metric will be sent only for those jobs which were completed earlier than the following threshold job_completion_threshold_time_minutes = 360 integrations: |- [integrations.azure_network_policy_manager] @@ -139,11 +139,12 @@ data: # Doc - https://github.com/microsoft/Docker-Provider/blob/ci_prod/Documentation/AgentSettings/ReadMe.md agent-settings: |- # prometheus scrape fluent bit settings for high scale - # buffer size should be greater than or equal to chunk size else we set it to chunk size. + # buffer size should be greater than or equal to chunk size else we set it to chunk size. [agent_settings.prometheus_fbit_settings] tcp_listener_chunk_size = 10 tcp_listener_buffer_size = 10 tcp_listener_mem_buf_limit = 200 + # The following settings are "undocumented", we don't recommend uncommenting them unless directed by Microsoft. # They increase the maximum stdout/stderr log collection rate but will also cause higher cpu/memory usage. # [agent_settings.fbit_config] diff --git a/kubernetes/linux/Dockerfile b/kubernetes/linux/Dockerfile index 2128d6d6a..becbe1157 100644 --- a/kubernetes/linux/Dockerfile +++ b/kubernetes/linux/Dockerfile @@ -17,13 +17,13 @@ ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9 RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg && rm -rf /var/lib/apt/lists/* COPY setup.sh main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar mdsd.xml envmdsd logrotate.conf $tmpdir/ -ARG IMAGE_TAG=ciprod03112022 +ARG IMAGE_TAG=ciprod03172022 ENV AGENT_VERSION ${IMAGE_TAG} WORKDIR ${tmpdir} # copy docker provider shell bundle to use the agent image -COPY ./Linux_ULINUX_1.0_x64_64_Release/docker-cimprov-*.*.*-*.x86_64.sh . +COPY ./Linux_ULINUX_1.0_*_64_Release/docker-cimprov-*.*.*-*.*.*.sh . # Note: If you prefer remote destination, uncomment below line and comment above line # wget https://github.com/microsoft/Docker-Provider/releases/download/10.0.0-1/docker-cimprov-10.0.0-1.universal.x86_64.sh diff --git a/kubernetes/linux/Dockerfile.multiarch b/kubernetes/linux/Dockerfile.multiarch new file mode 100644 index 000000000..df8b04d19 --- /dev/null +++ b/kubernetes/linux/Dockerfile.multiarch @@ -0,0 +1,39 @@ +FROM --platform=$BUILDPLATFORM golang:1.15.14 AS builder +ARG TARGETOS TARGETARCH +RUN /usr/bin/apt-get update && /usr/bin/apt-get install git g++ make pkg-config libssl-dev libpam0g-dev rpm librpm-dev uuid-dev libkrb5-dev python sudo gcc-aarch64-linux-gnu -y + +COPY build /src/build +COPY source /src/source +RUN cd /src/build/linux && make arch=${TARGETARCH} + + +FROM ubuntu:18.04 AS base_image +ARG TARGETOS TARGETARCH +MAINTAINER OMSContainers@microsoft.com +LABEL vendor=Microsoft\ Corp \ + com.microsoft.product="Azure Monitor for containers" +ENV tmpdir /opt +ENV APPLICATIONINSIGHTS_AUTH NzAwZGM5OGYtYTdhZC00NThkLWI5NWMtMjA3ZjM3NmM3YmRi +ENV MALLOC_ARENA_MAX 2 +ENV HOST_MOUNT_PREFIX /hostfs +ENV HOST_PROC /hostfs/proc +ENV HOST_SYS /hostfs/sys +ENV HOST_ETC /hostfs/etc +ENV HOST_VAR /hostfs/var +ENV AZMON_COLLECT_ENV False +ENV KUBE_CLIENT_BACKOFF_BASE 1 +ENV KUBE_CLIENT_BACKOFF_DURATION 0 +ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9 +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg make && rm -rf /var/lib/apt/lists/* + +COPY --from=builder /src/kubernetes/linux/Linux_ULINUX_1.0_*_64_Release/docker-cimprov-*.*.*-*.*.sh $tmpdir/ +COPY kubernetes/linux/setup.sh kubernetes/linux/main.sh kubernetes/linux/defaultpromenvvariables kubernetes/linux/defaultpromenvvariables-rs kubernetes/linux/defaultpromenvvariables-sidecar kubernetes/linux/mdsd.xml kubernetes/linux/envmdsd kubernetes/linux/logrotate.conf $tmpdir/ + +ARG IMAGE_TAG=ciprod03172022 +ENV AGENT_VERSION ${IMAGE_TAG} + +WORKDIR ${tmpdir} + +RUN chmod 775 $tmpdir/*.sh; sync; $tmpdir/setup.sh ${TARGETARCH} +CMD [ "/opt/main.sh" ] + diff --git a/kubernetes/linux/dockerbuild/build-and-publish-docker-image.sh b/kubernetes/linux/dockerbuild/build-and-publish-docker-image.sh index 267f15f32..580b158c9 100755 --- a/kubernetes/linux/dockerbuild/build-and-publish-docker-image.sh +++ b/kubernetes/linux/dockerbuild/build-and-publish-docker-image.sh @@ -14,6 +14,7 @@ usage() echo echo "Build and publish docker image:" echo "$basename --image " + echo "$basename --image --multiarch" } parse_args() @@ -30,6 +31,7 @@ for arg in "$@"; do shift case "$arg" in "--image") set -- "$@" "-i" ;; + "--multiarch") set -- "$@" "-m" ;; "--"*) usage ;; *) set -- "$@" "$arg" esac @@ -37,7 +39,7 @@ done local OPTIND opt -while getopts 'hi:' opt; do +while getopts 'hi:m' opt; do case "$opt" in h) usage @@ -48,6 +50,11 @@ while getopts 'hi:' opt; do echo "image is $OPTARG" ;; + m) + multi=1 + echo "using multiarch dockerfile" + ;; + ?) usage exit 1 @@ -131,6 +138,13 @@ echo "source code base directory: $baseDir" echo "build directory for docker provider: $buildDir" echo "docker file directory: $dockerFileDir" +if [ "$multi" -eq "1" ]; then + echo "building multiarch" + cd $baseDir + docker buildx build --platform linux/arm64/v8,linux/amd64 -t $image --build-arg IMAGE_TAG=$imageTag -f $linuxDir/Dockerfile.multiarch --push . + exit 0 +fi + # build docker provider shell bundle build_docker_provider diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index 2834bf972..997f624e2 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -175,7 +175,7 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then # convert the protocol prefix in lowercase for validation proxyprotocol=$(echo $proto | tr "[:upper:]" "[:lower:]") if [ "$proxyprotocol" != "http://" -a "$proxyprotocol" != "https://" ]; then - echo "-e error proxy endpoint should be in this format http(s)://:@:" + echo "-e error proxy endpoint should be in this format http(s)://: or http(s)://:@:" fi # remove the protocol url="$(echo ${PROXY_ENDPOINT/$proto/})" @@ -190,8 +190,8 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then # extract the port port="$(echo $hostport | sed -e 's,^.*:,:,g' -e 's,.*:\([0-9]*\).*,\1,g' -e 's,[^0-9],,g')" - if [ -z "$user" -o -z "$pwd" -o -z "$host" -o -z "$port" ]; then - echo "-e error proxy endpoint should be in this format http(s)://:@:" + if [ -z "$host" -o -z "$port" ]; then + echo "-e error proxy endpoint should be in this format http(s)://: or http(s)://:@:" else echo "successfully validated provided proxy endpoint is valid and expected format" fi @@ -202,18 +202,26 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then echo "export MDSD_PROXY_MODE=$MDSD_PROXY_MODE" >> ~/.bashrc export MDSD_PROXY_ADDRESS=$proto$hostport echo "export MDSD_PROXY_ADDRESS=$MDSD_PROXY_ADDRESS" >> ~/.bashrc - export MDSD_PROXY_USERNAME=$user - echo "export MDSD_PROXY_USERNAME=$MDSD_PROXY_USERNAME" >> ~/.bashrc - export MDSD_PROXY_PASSWORD_FILE=/opt/microsoft/docker-cimprov/proxy_password - echo "export MDSD_PROXY_PASSWORD_FILE=$MDSD_PROXY_PASSWORD_FILE" >> ~/.bashrc - #TODO: Compression + proxy creates a deserialization error in ODS. This needs a fix in MDSD - export MDSD_ODS_COMPRESSION_LEVEL=0 - echo "export MDSD_ODS_COMPRESSION_LEVEL=$MDSD_ODS_COMPRESSION_LEVEL" >> ~/.bashrc + if [ ! -z "$user" -a ! -z "$pwd" ]; then + export MDSD_PROXY_USERNAME=$user + echo "export MDSD_PROXY_USERNAME=$MDSD_PROXY_USERNAME" >> ~/.bashrc + export MDSD_PROXY_PASSWORD_FILE=/opt/microsoft/docker-cimprov/proxy_password + echo "export MDSD_PROXY_PASSWORD_FILE=$MDSD_PROXY_PASSWORD_FILE" >> ~/.bashrc + fi + if [ -e "/etc/omsagent-secret/PROXYCERT.crt" ]; then + export PROXY_CA_CERT=/etc/omsagent-secret/PROXYCERT.crt + echo "export PROXY_CA_CERT=$PROXY_CA_CERT" >> ~/.bashrc + fi fi if [ ! -z "$PROXY_ENDPOINT" ]; then - echo "Making curl request to oms endpint with domain: $domain and proxy: $PROXY_ENDPOINT" - curl --max-time 10 https://$workspaceId.oms.$domain/AgentService.svc/LinuxAgentTopologyRequest --proxy $PROXY_ENDPOINT + if [ -e "/etc/omsagent-secret/PROXYCERT.crt" ]; then + echo "Making curl request to oms endpint with domain: $domain and proxy endpoint, and proxy CA cert" + curl --max-time 10 https://$workspaceId.oms.$domain/AgentService.svc/LinuxAgentTopologyRequest --proxy $PROXY_ENDPOINT --proxy-cacert /etc/omsagent-secret/PROXYCERT.crt + else + echo "Making curl request to oms endpint with domain: $domain and proxy endpoint" + curl --max-time 10 https://$workspaceId.oms.$domain/AgentService.svc/LinuxAgentTopologyRequest --proxy $PROXY_ENDPOINT + fi else echo "Making curl request to oms endpint with domain: $domain" curl --max-time 10 https://$workspaceId.oms.$domain/AgentService.svc/LinuxAgentTopologyRequest @@ -221,8 +229,13 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then if [ $? -ne 0 ]; then if [ ! -z "$PROXY_ENDPOINT" ]; then - echo "Making curl request to ifconfig.co with proxy: $PROXY_ENDPOINT" - RET=`curl --max-time 10 -s -o /dev/null -w "%{http_code}" ifconfig.co --proxy $PROXY_ENDPOINT` + if [ -e "/etc/omsagent-secret/PROXYCERT.crt" ]; then + echo "Making curl request to ifconfig.co with proxy and proxy CA cert" + RET=`curl --max-time 10 -s -o /dev/null -w "%{http_code}" ifconfig.co --proxy $PROXY_ENDPOINT --proxy-cacert /etc/omsagent-secret/PROXYCERT.crt` + else + echo "Making curl request to ifconfig.co with proxy" + RET=`curl --max-time 10 -s -o /dev/null -w "%{http_code}" ifconfig.co --proxy $PROXY_ENDPOINT` + fi else echo "Making curl request to ifconfig.co" RET=`curl --max-time 10 -s -o /dev/null -w "%{http_code}" ifconfig.co` @@ -232,8 +245,13 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then else # Retrying here to work around network timing issue if [ ! -z "$PROXY_ENDPOINT" ]; then - echo "ifconfig check succeeded, retrying oms endpoint with proxy..." - curl --max-time 10 https://$workspaceId.oms.$domain/AgentService.svc/LinuxAgentTopologyRequest --proxy $PROXY_ENDPOINT + if [ -e "/etc/omsagent-secret/PROXYCERT.crt" ]; then + echo "ifconfig check succeeded, retrying oms endpoint with proxy and proxy CA cert..." + curl --max-time 10 https://$workspaceId.oms.$domain/AgentService.svc/LinuxAgentTopologyRequest --proxy $PROXY_ENDPOINT --proxy-cacert /etc/omsagent-secret/PROXYCERT.crt + else + echo "ifconfig check succeeded, retrying oms endpoint with proxy..." + curl --max-time 10 https://$workspaceId.oms.$domain/AgentService.svc/LinuxAgentTopologyRequest --proxy $PROXY_ENDPOINT + fi else echo "ifconfig check succeeded, retrying oms endpoint..." curl --max-time 10 https://$workspaceId.oms.$domain/AgentService.svc/LinuxAgentTopologyRequest @@ -310,7 +328,7 @@ source ~/.bashrc if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then #Parse the configmap to set the right environment variables. - /usr/bin/ruby2.6 tomlparser.rb + /usr/bin/ruby2.7 tomlparser.rb cat config_env_var | while read line; do echo $line >> ~/.bashrc @@ -321,7 +339,7 @@ fi #Parse the configmap to set the right environment variables for agent config. #Note > tomlparser-agent-config.rb has to be parsed first before td-agent-bit-conf-customizer.rb for fbit agent settings if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then - /usr/bin/ruby2.6 tomlparser-agent-config.rb + /usr/bin/ruby2.7 tomlparser-agent-config.rb cat agent_config_env_var | while read line; do #echo $line @@ -330,7 +348,7 @@ if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then source agent_config_env_var #Parse the configmap to set the right environment variables for network policy manager (npm) integration. - /usr/bin/ruby2.6 tomlparser-npm-config.rb + /usr/bin/ruby2.7 tomlparser-npm-config.rb cat integration_npm_config_env_var | while read line; do #echo $line @@ -341,11 +359,11 @@ fi #Replace the placeholders in td-agent-bit.conf file for fluentbit with custom/default values in daemonset if [ ! -e "/etc/config/kube.conf" ] && [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then - /usr/bin/ruby2.6 td-agent-bit-conf-customizer.rb + /usr/bin/ruby2.7 td-agent-bit-conf-customizer.rb fi #Parse the prometheus configmap to create a file with new custom settings. -/usr/bin/ruby2.6 tomlparser-prom-customconfig.rb +/usr/bin/ruby2.7 tomlparser-prom-customconfig.rb #Setting default environment variables to be used in any case of failure in the above steps if [ ! -e "/etc/config/kube.conf" ]; then @@ -379,7 +397,7 @@ fi if [ ! -e "/etc/config/kube.conf" ]; then if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then #Parse the agent configmap to create a file with new custom settings. - /usr/bin/ruby2.6 tomlparser-prom-agent-config.rb + /usr/bin/ruby2.7 tomlparser-prom-agent-config.rb #Sourcing config environment variable file if it exists if [ -e "side_car_fbit_config_env_var" ]; then cat side_car_fbit_config_env_var | while read line; do @@ -393,7 +411,7 @@ fi #Parse the configmap to set the right environment variables for MDM metrics configuration for Alerting. if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then - /usr/bin/ruby2.6 tomlparser-mdm-metrics-config.rb + /usr/bin/ruby2.7 tomlparser-mdm-metrics-config.rb cat config_mdm_metrics_env_var | while read line; do echo $line >> ~/.bashrc @@ -401,7 +419,7 @@ if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then source config_mdm_metrics_env_var #Parse the configmap to set the right environment variables for metric collection settings - /usr/bin/ruby2.6 tomlparser-metric-collection-config.rb + /usr/bin/ruby2.7 tomlparser-metric-collection-config.rb cat config_metric_collection_env_var | while read line; do echo $line >> ~/.bashrc @@ -412,7 +430,7 @@ fi # OSM scraping to be done in replicaset if sidecar car scraping is disabled and always do the scraping from the sidecar (It will always be either one of the two) if [[ ( ( ! -e "/etc/config/kube.conf" ) && ( "${CONTAINER_TYPE}" == "PrometheusSidecar" ) ) || ( ( -e "/etc/config/kube.conf" ) && ( "${SIDECAR_SCRAPING_ENABLED}" == "false" ) ) ]]; then - /usr/bin/ruby2.6 tomlparser-osm-config.rb + /usr/bin/ruby2.7 tomlparser-osm-config.rb if [ -e "integration_osm_config_env_var" ]; then cat integration_osm_config_env_var | while read line; do @@ -499,7 +517,7 @@ if [ "$CONTAINER_RUNTIME" != "docker" ]; then fi echo "set caps for ruby process to read container env from proc" -sudo setcap cap_sys_ptrace,cap_dac_read_search+ep /usr/bin/ruby2.6 +sudo setcap cap_sys_ptrace,cap_dac_read_search+ep /usr/bin/ruby2.7 echo "export KUBELET_RUNTIME_OPERATIONS_METRIC="$KUBELET_RUNTIME_OPERATIONS_METRIC >> ~/.bashrc echo "export KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC="$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC >> ~/.bashrc @@ -562,6 +580,13 @@ else echo "export CIWORKSPACE_keyFile=$CIWORKSPACE_keyFile" >> ~/.bashrc export MDSD_FLUENT_SOCKET_PORT="29230" echo "export MDSD_FLUENT_SOCKET_PORT=$MDSD_FLUENT_SOCKET_PORT" >> ~/.bashrc + # set the libcurl specific env and configuration + export ENABLE_CURL_UPLOAD=true + echo "export ENABLE_CURL_UPLOAD=$ENABLE_CURL_UPLOAD" >> ~/.bashrc + export CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + echo "export CURL_CA_BUNDLE=$CURL_CA_BUNDLE" >> ~/.bashrc + mkdir -p /etc/pki/tls/certs + cp /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt fi source ~/.bashrc diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 872ac99cf..00d5bc0fa 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -1,6 +1,12 @@ TMPDIR="/opt" cd $TMPDIR +if [ -z $1 ]; then + ARCH="amd64" +else + ARCH=$1 +fi + #Download utf-8 encoding capability on the omsagent container. #upgrade apt to latest version apt-get update && apt-get install -y apt && DEBIAN_FRONTEND=noninteractive apt-get install -y locales @@ -9,14 +15,18 @@ sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ update-locale LANG=en_US.UTF-8 -#install oneagent - Official bits (10/7/2021) -wget https://github.com/microsoft/Docker-Provider/releases/download/1.14/azure-mdsd_1.14.2-build.master.284_x86_64.deb +#install oneagent - Official bits (3/14/2022) +if [ "${ARCH}" != "arm64" ]; then + wget "https://github.com/microsoft/Docker-Provider/releases/download/1.17.0/azure-mdsd_1.17.0-build.master.354_x86_64.deb" -O azure-mdsd.deb +else + wget "https://github.com/microsoft/Docker-Provider/releases/download/1.17.1-arm64-master/azure-mdsd_1.17.1-build.master.366_aarch64.deb" -O azure-mdsd.deb +fi /usr/bin/dpkg -i $TMPDIR/azure-mdsd*.deb cp -f $TMPDIR/mdsd.xml /etc/mdsd.d cp -f $TMPDIR/envmdsd /etc/mdsd.d -#log rotate conf for mdsd and can be extended for other log files as well +# log rotate conf for mdsd and can be extended for other log files as well cp -f $TMPDIR/logrotate.conf /etc/logrotate.d/ci-agent #download inotify tools for watching configmap changes @@ -30,27 +40,27 @@ sudo apt-get install jq=1.5+dfsg-2 -y #used to setcaps for ruby process to read /proc/env sudo apt-get install libcap2-bin -y -wget https://dl.influxdata.com/telegraf/releases/telegraf-1.20.3_linux_amd64.tar.gz -tar -zxvf telegraf-1.20.3_linux_amd64.tar.gz +wget https://dl.influxdata.com/telegraf/releases/telegraf-1.20.3_linux_$ARCH.tar.gz +tar -zxvf telegraf-1.20.3_linux_$ARCH.tar.gz mv /opt/telegraf-1.20.3/usr/bin/telegraf /opt/telegraf -chmod 777 /opt/telegraf +chmod 544 /opt/telegraf # Use wildcard version so that it doesnt require to touch this file -/$TMPDIR/docker-cimprov-*.*.*-*.x86_64.sh --install +/$TMPDIR/docker-cimprov-*.*.*-*.*.sh --install #download and install fluent-bit(td-agent-bit) wget -qO - https://packages.fluentbit.io/fluentbit.key | sudo apt-key add - -sudo echo "deb https://packages.fluentbit.io/ubuntu/xenial xenial main" >> /etc/apt/sources.list +sudo echo "deb https://packages.fluentbit.io/ubuntu/bionic bionic main" >> /etc/apt/sources.list sudo apt-get update sudo apt-get install td-agent-bit=1.7.8 -y -# install ruby2.6 +# install ruby2.7 sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F5DA5F09C3173AA6 sudo echo "deb http://ppa.launchpad.net/brightbox/ruby-ng/ubuntu bionic main" >> /etc/apt/sources.list sudo apt-get update -sudo apt-get install ruby2.6 ruby2.6-dev gcc make -y +sudo apt-get install ruby2.7 ruby2.7-dev gcc make -y # fluentd v1 gem gem install fluentd -v "1.14.2" --no-document fluentd --setup ./fluent @@ -64,7 +74,7 @@ rm -f $TMPDIR/envmdsd rm -f $TMPDIR/telegraf-*.tar.gz # remove build dependencies -sudo apt-get remove ruby2.6-dev gcc make -y +sudo apt-get remove ruby2.7-dev gcc make -y # Remove settings for cron.daily that conflict with the node's cron.daily. Since both are trying to rotate the same files # in /var/log at the same time, the rotation doesn't happen correctly and then the *.1 file is forever logged to. diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index 37b9a020a..c8324370b 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -357,7 +357,7 @@ spec: component: oms-agent tier: node annotations: - agentVersion: "azure-mdsd-1.14.2" + agentVersion: "azure-mdsd-1.17.0" dockerProviderVersion: "16.0.0-0" schema-versions: "v1" spec: @@ -368,7 +368,7 @@ spec: value: "3" containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03112022" + image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03172022" imagePullPolicy: IfNotPresent resources: limits: @@ -453,65 +453,65 @@ spec: periodSeconds: 60 timeoutSeconds: 15 #Only in sidecar scraping mode - # - name: omsagent-prometheus - # image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03112022" - # imagePullPolicy: IfNotPresent - # resources: - # limits: - # cpu: 500m - # memory: 1Gi - # requests: - # cpu: 75m - # memory: 225Mi - # env: - # # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these - # - name: AKS_CLUSTER_NAME - # value: "VALUE_AKS_CLUSTER_NAME" - # - name: AKS_RESOURCE_ID - # value: "VALUE_AKS_RESOURCE_ID_VALUE" - # - name: AKS_REGION - # value: "VALUE_AKS_RESOURCE_REGION_VALUE" - # - name: AKS_NODE_RESOURCE_GROUP - # value: "VALUE_AKS_NODE_RESOURCE_GROUP" - # #Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters - # #- name: ACS_RESOURCE_NAME - # # value: "my_acs_cluster_name" - # - name: CONTAINER_TYPE - # value: "PrometheusSidecar" - # - name: CONTROLLER_TYPE - # value: "DaemonSet" - # - name: NODE_IP - # valueFrom: - # fieldRef: - # fieldPath: status.hostIP - # # Update this with the user assigned msi client id for omsagent - # - name: USER_ASSIGNED_IDENTITY_CLIENT_ID - # value: "VALUE_USER_ASSIGNED_IDENTITY_CLIENT_ID_VALUE" - # - name: USING_AAD_MSI_AUTH - # value: "false" - # securityContext: - # privileged: true - # volumeMounts: - # - mountPath: /etc/kubernetes/host - # name: azure-json-path - # - mountPath: /etc/omsagent-secret - # name: omsagent-secret - # readOnly: true - # - mountPath: /etc/config/settings - # name: settings-vol-config - # readOnly: true - # - mountPath: /etc/config/osm-settings - # name: osm-settings-vol-config - # readOnly: true - # livenessProbe: - # exec: - # command: - # - /bin/bash - # - -c - # - /opt/livenessprobe.sh - # initialDelaySeconds: 60 - # periodSeconds: 60 - # timeoutSeconds: 15 + - name: omsagent-prometheus + image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03172022" + imagePullPolicy: IfNotPresent + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 75m + memory: 225Mi + env: + # azure devops pipeline uses AKS_RESOURCE_ID and AKS_REGION hence ensure to uncomment these + - name: AKS_CLUSTER_NAME + value: "VALUE_AKS_CLUSTER_NAME" + - name: AKS_RESOURCE_ID + value: "VALUE_AKS_RESOURCE_ID_VALUE" + - name: AKS_REGION + value: "VALUE_AKS_RESOURCE_REGION_VALUE" + - name: AKS_NODE_RESOURCE_GROUP + value: "VALUE_AKS_NODE_RESOURCE_GROUP" + #Uncomment below two lines for ACS clusters and set the cluster names manually. Also comment out the above two lines for ACS clusters + #- name: ACS_RESOURCE_NAME + # value: "my_acs_cluster_name" + - name: CONTAINER_TYPE + value: "PrometheusSidecar" + - name: CONTROLLER_TYPE + value: "DaemonSet" + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + # Update this with the user assigned msi client id for omsagent + - name: USER_ASSIGNED_IDENTITY_CLIENT_ID + value: "VALUE_USER_ASSIGNED_IDENTITY_CLIENT_ID_VALUE" + - name: USING_AAD_MSI_AUTH + value: "false" + securityContext: + privileged: true + volumeMounts: + - mountPath: /etc/kubernetes/host + name: azure-json-path + - mountPath: /etc/omsagent-secret + name: omsagent-secret + readOnly: true + - mountPath: /etc/config/settings + name: settings-vol-config + readOnly: true + - mountPath: /etc/config/osm-settings + name: osm-settings-vol-config + readOnly: true + livenessProbe: + exec: + command: + - /bin/bash + - -c + - /opt/livenessprobe.sh + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 15 affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -596,14 +596,14 @@ spec: labels: rsName: "omsagent-rs" annotations: - agentVersion: "azure-mdsd-1.14.2" + agentVersion: "azure-mdsd-1.17.0" dockerProviderVersion: "16.0.0-0" schema-versions: "v1" spec: serviceAccountName: omsagent containers: - name: omsagent - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03112022" + image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod03172022" imagePullPolicy: IfNotPresent resources: limits: @@ -634,7 +634,7 @@ spec: value: "VALUE_USER_ASSIGNED_IDENTITY_CLIENT_ID_VALUE" # Add the below environment variable to true only in sidecar enabled regions, else set it to false - name: SIDECAR_SCRAPING_ENABLED - value: "false" + value: "true" - name: USING_AAD_MSI_AUTH value: "false" securityContext: @@ -776,7 +776,7 @@ spec: value: "3" containers: - name: omsagent-win - image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod03112022" + image: "mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod03172022" imagePullPolicy: IfNotPresent resources: limits: @@ -811,7 +811,7 @@ spec: fieldRef: fieldPath: status.hostIP - name: SIDECAR_SCRAPING_ENABLED - value: "false" + value: "true" # Update this with the user assigned msi client id for omsagent - name: USER_ASSIGNED_IDENTITY_CLIENT_ID value: "VALUE_USER_ASSIGNED_IDENTITY_CLIENT_ID_VALUE" diff --git a/kubernetes/windows/Dockerfile b/kubernetes/windows/Dockerfile index 26adfdad1..3aa4054e1 100644 --- a/kubernetes/windows/Dockerfile +++ b/kubernetes/windows/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER OMSContainers@microsoft.com LABEL vendor=Microsoft\ Corp \ com.microsoft.product="Azure Monitor for containers" -ARG IMAGE_TAG=win-ciprod03112022 +ARG IMAGE_TAG=win-ciprod03172022 # Do not split this into multiple RUN! # Docker creates a layer for every RUN-Statement @@ -57,9 +57,6 @@ COPY ./omsagentwindows/out_oms.so /opt/omsagentwindows/out_oms.so # copy fluent, fluent-bit and out_oms conf files COPY ./omsagentwindows/installer/conf/fluent.conf /etc/fluent/ -# copy fluent docker and cri parser conf files -COPY ./omsagentwindows/installer/conf/fluent-cri-parser.conf /etc/fluent/ -COPY ./omsagentwindows/installer/conf/fluent-docker-parser.conf /etc/fluent/ COPY ./omsagentwindows/installer/conf/fluent-bit.conf /etc/fluent-bit COPY ./omsagentwindows/installer/conf/azm-containers-parser.conf /etc/fluent-bit/ COPY ./omsagentwindows/installer/conf/out_oms.conf /etc/omsagentwindows diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 106e94c0f..6482daed9 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -120,12 +120,12 @@ function Set-EnvironmentVariables { $proxy = [string]$proxy.Trim(); $parts = $proxy -split "@" if ($parts.Length -ne 2) { - Write-Host "Invalid ProxyConfiguration $($proxy). EXITING....." + Write-Host "Invalid ProxyConfiguration. EXITING....." exit 1 } $subparts1 = $parts[0] -split "//" if ($subparts1.Length -ne 2) { - Write-Host "Invalid ProxyConfiguration $($proxy). EXITING....." + Write-Host "Invalid ProxyConfiguration. EXITING....." exit 1 } $protocol = $subparts1[0].ToLower().TrimEnd(":") @@ -133,16 +133,18 @@ function Set-EnvironmentVariables { Write-Host "Unsupported protocol in ProxyConfiguration $($proxy). EXITING....." exit 1 } - $subparts2 = $parts[1] -split ":" - if ($subparts2.Length -ne 2) { - Write-Host "Invalid ProxyConfiguration $($proxy). EXITING....." - exit 1 - } + } } + Write-Host "Provided Proxy configuration is valid" } + if (Test-Path /etc/omsagent-secret/PROXYCERT.crt) { + Write-Host "Importing Proxy CA cert since Proxy CA cert configured" + Import-Certificate -FilePath /etc/omsagent-secret/PROXYCERT.crt -CertStoreLocation 'Cert:\LocalMachine\Root' -Verbose + } + # Set PROXY [System.Environment]::SetEnvironmentVariable("PROXY", $proxy, "Process") [System.Environment]::SetEnvironmentVariable("PROXY", $proxy, "Machine") @@ -196,6 +198,7 @@ function Set-EnvironmentVariables { Write-Host $_.Exception } } + # Check if the fetched IKey was properly encoded. if not then turn off telemetry if ($aiKeyFetched -match '^[A-Za-z0-9=]+$') { Write-Host "Using cloud-specific instrumentation key" @@ -302,7 +305,7 @@ function Set-EnvironmentVariables { # run config parser ruby /opt/omsagentwindows/scripts/ruby/tomlparser.rb .\setenv.ps1 - + #Parse the configmap to set the right environment variables for agent config. ruby /opt/omsagentwindows/scripts/ruby/tomlparser-agent-config.rb .\setagentenv.ps1 @@ -436,8 +439,6 @@ function Start-Fluent-Telegraf { # Run fluent-bit as a background job. Switch this to a windows service once fluent-bit supports natively running as a windows service Start-Job -ScriptBlock { Start-Process -NoNewWindow -FilePath "C:\opt\fluent-bit\bin\fluent-bit.exe" -ArgumentList @("-c", "C:\etc\fluent-bit\fluent-bit.conf", "-e", "C:\opt\omsagentwindows\out_oms.so") } - $containerRuntime = Get-ContainerRuntime - #register fluentd as a service and start # there is a known issues with win32-service https://github.com/chef/win32-service/issues/70 if (![string]::IsNullOrEmpty($containerRuntime) -and [string]$containerRuntime.StartsWith('docker') -eq $false) { @@ -613,4 +614,4 @@ Start-Fluent-Telegraf Get-WmiObject Win32_process | Where-Object { $_.Name -match 'powershell' } | Format-Table -Property Name, CommandLine, ProcessId #check if fluentd service is running -Get-Service fluentdwinaks +Get-Service fluentdwinaks \ No newline at end of file diff --git a/scripts/build/linux/install-build-pre-requisites.sh b/scripts/build/linux/install-build-pre-requisites.sh index 385d6fffc..b85e54fc4 100644 --- a/scripts/build/linux/install-build-pre-requisites.sh +++ b/scripts/build/linux/install-build-pre-requisites.sh @@ -16,7 +16,7 @@ install_go_lang() sudo tar -xvf go1.15.14.linux-amd64.tar.gz sudo mv -f go /usr/local echo "set file permission for go bin" - sudo chmod 777 /usr/local/go/bin + sudo chmod 744 /usr/local/go/bin echo "installation of go 1.15.14 completed." echo "installation of go 1.15.14 completed." fi @@ -58,10 +58,26 @@ install_docker() sudo apt-get install docker-ce docker-ce-cli containerd.io -y # Allow your user to access the Docker CLI without needing root access. sudo usermod -aG docker $USER + newgrp docker echo "installing docker completed" fi } +install_docker_buildx() +{ + # install the buildx plugin + sudo curl -O https://github.com/docker/buildx/releases/download/v0.7.1/buildx-v0.7.1.linux-amd64 + sudo mkdir -p $HOME/.docker/cli-plugins + sudo mv buildx-v* $HOME/.docker/cli-plugins + + # install the emulator support + sudo apt-get -y install qemu binfmt-support qemu-user-static + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + docker buildx create --name testbuilder + docker buildx use testbuilder +} + install_python() { echo "installing python ..." @@ -125,6 +141,9 @@ install_build_dependencies # install docker install_docker +# install buildx +install_docker_buildx + # install go install_go_lang diff --git a/scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterOnboarding.json b/scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterOnboarding.json index c77e3203d..d5b613537 100644 --- a/scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterOnboarding.json +++ b/scripts/onboarding/aks/onboarding-using-msi-auth/existingClusterOnboarding.json @@ -35,7 +35,7 @@ "dcrResourceTagValues": { "type": "object", "metadata": { - "description": "Existing or new tags on DCR Cluster Resource" + "description": "Existing or new tags on DCR Resource" } } }, @@ -43,9 +43,10 @@ "clusterSubscriptionId": "[split(parameters('aksResourceId'),'/')[2]]", "clusterResourceGroup": "[split(parameters('aksResourceId'),'/')[4]]", "clusterName": "[split(parameters('aksResourceId'),'/')[8]]", + "clusterLocation": "[replace(parameters('aksResourceLocation'),' ', '')]", "workspaceSubscriptionId": "[split(parameters('workspaceResourceId'),'/')[2]]", "workspaceResourceGroup": "[split(parameters('workspaceResourceId'),'/')[4]]", - "dcrName": "[Concat('MSCI', '-', split(parameters('workspaceResourceId'),'/')[8])]", + "dcrName": "[Concat('MSCI', '-', variables('clusterName'), '-', variables('clusterLocation'))]", "associationName": "ContainerInsightsExtension", "dataCollectionRuleId": "[resourceId(variables('workspaceSubscriptionId'), variables('workspaceResourceGroup'), 'Microsoft.Insights/dataCollectionRules', variables('dcrName'))]" }, diff --git a/scripts/onboarding/managed/enable-monitoring.ps1 b/scripts/onboarding/managed/enable-monitoring.ps1 index b72e10e02..27bc2fd62 100644 --- a/scripts/onboarding/managed/enable-monitoring.ps1 +++ b/scripts/onboarding/managed/enable-monitoring.ps1 @@ -250,13 +250,6 @@ if (([string]::IsNullOrEmpty($servicePrincipalClientId) -eq $false) -and $isUsingServicePrincipal = $true } -if (([string]::IsNullOrEmpty($servicePrincipalClientId) -eq $false) -and - ([string]::IsNullOrEmpty($servicePrincipalClientSecret) -eq $false) -and - ([string]::IsNullOrEmpty($tenantId) -eq $false)) { - Write-Host("Using service principal creds for the azure login since these provided.") - $isUsingServicePrincipal = $true -} - if ($clusterResourceId.ToLower().Contains("microsoft.kubernetes/connectedclusters") -eq $true) { $isArcK8sCluster = $true } diff --git a/source/plugins/go/src/Makefile b/source/plugins/go/src/Makefile index b3b730d79..f9488bc8e 100644 --- a/source/plugins/go/src/Makefile +++ b/source/plugins/go/src/Makefile @@ -1,6 +1,16 @@ BASE_DIR := $(subst /build/linux,,$(PWD)) include $(BASE_DIR)/build/version +ifeq ($(arch),) + PF_ARCH=amd64 +else + PF_ARCH=$(arch) +endif + +ifneq ($(PF_ARCH),amd64) + OPTIONS=CGO_ENABLED=1 CC=aarch64-linux-gnu-gcc GOOS=linux GOARCH=arm64 +endif + fbplugin: @echo "========================= Building out_oms plugin go code =========================" export BUILDVERSION=$(CONTAINER_BUILDVERSION_MAJOR).$(CONTAINER_BUILDVERSION_MINOR).$(CONTAINER_BUILDVERSION_PATCH)-$(CONTAINER_BUILDVERSION_BUILDNR) @@ -12,7 +22,7 @@ fbplugin: @echo "========================= go get =========================" go get @echo "========================= go build =========================" - go build -ldflags "-X 'main.revision=$(BUILDVERSION)' -X 'main.builddate=$(BUILDDATE)'" -buildmode=c-shared -o out_oms.so . + $(OPTIONS) go build -ldflags "-X 'main.revision=$(BUILDVERSION)' -X 'main.builddate=$(BUILDDATE)'" -buildmode=c-shared -o out_oms.so . test: go test -cover -race -coverprofile=coverage.txt -covermode=atomic diff --git a/source/plugins/go/src/go.mod b/source/plugins/go/src/go.mod index e930238c2..9f30afab1 100644 --- a/source/plugins/go/src/go.mod +++ b/source/plugins/go/src/go.mod @@ -3,10 +3,8 @@ module Docker-Provider/source/plugins/go/src go 1.14 require ( - github.com/Azure/azure-kusto-go v0.3.2 - github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 - github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect - github.com/dnaeon/go-vcr v1.2.0 // indirect + github.com/Azure/azure-kusto-go v0.5.2 + github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 github.com/fluent/fluent-bit-go v0.0.0-20171103221316-c4a158a6e3a7 github.com/golang/mock v1.4.1 github.com/google/uuid v1.3.0 diff --git a/source/plugins/go/src/go.sum b/source/plugins/go/src/go.sum index 53a853e83..c5d7ea147 100644 --- a/source/plugins/go/src/go.sum +++ b/source/plugins/go/src/go.sum @@ -23,54 +23,42 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c h1:5eeuG0BHx1+DHeT3AP+ISKZ2ht1UjGhm581ljqYpVeQ= code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-kusto-go v0.3.2 h1:XpS9co6GvEDl2oICF9HsjEsQVwEpRK6wbNWb9Z+uqsY= -github.com/Azure/azure-kusto-go v0.3.2/go.mod h1:wd50n4qlsSxh+G4f80t+Fnl2ShK9AcXD+lMOstiKuYo= +github.com/Azure/azure-kusto-go v0.5.2 h1:6kFVZp4iyz8YFTuxrIdivAXVcEs5wNKTVK5gai+E8pk= +github.com/Azure/azure-kusto-go v0.5.2/go.mod h1:2xOhBxRcHyyNifFHmNMcqYL6AMdhyrUHCkEJkrZ+EI4= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= -github.com/Azure/azure-pipeline-go v0.2.1 h1:OLBdZJ3yvOn2MezlWvbrBMTEUQC72zAftRZOMdj5HYo= -github.com/Azure/azure-pipeline-go v0.2.1/go.mod h1:UGSo8XybXnIGZ3epmeBw7Jdz+HiUVpqIlpz/HKHylF4= -github.com/Azure/azure-sdk-for-go v44.1.0+incompatible h1:l1UGvaaoMCUwVGUauvHzeB4t+Y0yPX5iJwBhzc0LqyE= -github.com/Azure/azure-sdk-for-go v44.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFEY95rZLK0Tj6o= -github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-sdk-for-go v61.2.0+incompatible h1:sSormXkfW0ov1vh6ihTBRQxdfg73fPqkccl50GbR9iM= +github.com/Azure/azure-sdk-for-go v61.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= +github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd h1:b3wyxBl3vvr15tUAziPBPK354y+LSdfPCpex5oBttHo= github.com/Azure/azure-storage-queue-go v0.0.0-20191125232315-636801874cdd/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= -github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= -github.com/Azure/go-autorest/autorest v0.10.0 h1:mvdtztBqcL8se7MdrUweNieTNi4kfNG6GOJuurQJpuY= -github.com/Azure/go-autorest/autorest v0.10.0/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest v0.11.12 h1:gI8ytXbxMfI+IVbI9mP2JGCTXIuhHLgRlvQ9X4PsnHE= github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= -github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= -github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= -github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.8.2 h1:O1X4oexUxnZCaEUGsvMnr8ZGj8HI37tNezwY4npRqA0= -github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest v0.11.24 h1:1fIGgHKqVm54KIPT+q8Zmd1QlVsmHqeUGso5qm2BqqE= +github.com/Azure/go-autorest/autorest v0.11.24/go.mod h1:G6kyRlFnTuSbEYkQGawPfsCswgme4iYf6rfSKUDzbCc= github.com/Azure/go-autorest/autorest/adal v0.9.5 h1:Y3bBUV4rTuxenJJs41HU3qmqsb+auo+a3Lz+PlJPpL0= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= -github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 h1:iM6UAvjR97ZIeR93qTcwpKNMpV+/FTWjwEbuPD495Tk= -github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM= -github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 h1:LXl088ZQlP0SBppGFsRZonW6hSvwgL5gRByMbvUbx8U= -github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw= -github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= -github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= -github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 h1:P6bYXFoao05z5uhOQzbC3Qd8JqF3jUoocoTeIxkp2cA= +github.com/Azure/go-autorest/autorest/azure/auth v0.5.11/go.mod h1:84w/uV8E37feW2NCJ08uT9VBfjfUHpgLVnG2InYD6cg= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 h1:0W/yGmFdTIT77fvdlGZ0LMISoLHFJ7Tx4U0yeB+uFs4= +github.com/Azure/go-autorest/autorest/azure/cli v0.4.5/go.mod h1:ADQAXrkgm7acgWVUNamOgh8YNrv4p27l3Wc55oVfpzg= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= -github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= -github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.4.0 h1:oXVqrxakqqV1UZdSazDOPOLvOIz+XA683u8EctwboHk= github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= -github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= -github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= github.com/Azure/go-autorest/logger v0.2.0 h1:e4RVHVZKC5p6UANLJHkM4OfR1UKZPj8Wt8Pcx+3oqrE= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= -github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= -github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -89,10 +77,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= -github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= +github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= @@ -125,6 +111,9 @@ github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZg github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0 h1:besgBTC8w8HjP6NzQdxwKH9Z5oQMZ24ThTrHp3cZ8eU= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -197,6 +186,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -206,8 +197,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149 h1:HfxbT6/JcvIljmERptWhwa8XzP7H3T+Z2N26gTsaDaA= -github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/microsoft/ApplicationInsights-Go v0.4.3 h1:gBuy5rM3o6Zo69QTkq1Ens8wx6sVf+mpgMjjfayiRcw= github.com/microsoft/ApplicationInsights-Go v0.4.3/go.mod h1:ih0t3h84PdzV1qGeUs89o9wL8eCuwf24M7TZp/nyqXk= github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= @@ -274,12 +265,13 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= -golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 h1:0es+/5331RGQPcXlMfP+WrnIIS6dNnNRe0WB02W0F4M= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -324,6 +316,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -334,6 +327,9 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7 h1:OgUuv8lsRpBibGNbSizVwKWlysjaNzmC9gYMhPVfqFM= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 h1:CIJ76btIcR3eFI5EgSo6k1qKw9KJexJuRLI9G7Hp5wE= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -360,6 +356,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -370,10 +367,14 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073 h1:8qxJSnu+7dRq6upnbntrmriWByIakBuct5OM/MdQC1M= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -387,6 +388,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -488,6 +491,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index 407ab3611..99329ac6f 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -1713,7 +1713,7 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { // Try to read the ADX database name from environment variables. Default to DefaultAdsDatabaseName if not set. // This SHOULD be set by tomlparser.rb so it's a highly unexpected event if it isn't. // It should be set by the logic in tomlparser.rb EVEN if ADX logging isn't enabled - AdxDatabaseName := strings.TrimSpace(os.Getenv("AZMON_ADX_DATABASE_NAME")) + AdxDatabaseName = strings.TrimSpace(os.Getenv("AZMON_ADX_DATABASE_NAME")) // Check the len of the provided name for database and use default if 0, just to be sure if len(AdxDatabaseName) == 0 { diff --git a/source/plugins/go/src/utils.go b/source/plugins/go/src/utils.go index 9ca8980db..23353a372 100644 --- a/source/plugins/go/src/utils.go +++ b/source/plugins/go/src/utils.go @@ -192,6 +192,7 @@ func CreateADXClient() { //log.Fatalf("Unable to create ADX connection %s", err.Error()) } else { Log("Successfully created ADX Client. Creating Ingestor...") + Log("AdxDatabaseName=%s", AdxDatabaseName) ingestor, ingestorErr := ingest.New(client, AdxDatabaseName, "ContainerLogV2") if ingestorErr != nil { Log("Error::mdsd::Unable to create ADX ingestor %s", ingestorErr.Error()) @@ -232,6 +233,7 @@ func isValidUrl(uri string) bool { func convertMsgPackEntriesToMsgpBytes(fluentForwardTag string, msgPackEntries []MsgPackEntry) []byte { var msgpBytes []byte + fluentForward := MsgPackForward{ Tag: fluentForwardTag, Entries: msgPackEntries, diff --git a/source/plugins/ruby/ApplicationInsightsUtility.rb b/source/plugins/ruby/ApplicationInsightsUtility.rb index eb143c4ba..70d0a400e 100644 --- a/source/plugins/ruby/ApplicationInsightsUtility.rb +++ b/source/plugins/ruby/ApplicationInsightsUtility.rb @@ -79,6 +79,9 @@ def initializeUtility() $log.info("proxy configured") @@CustomProperties["IsProxyConfigured"] = "true" isProxyConfigured = true + if ProxyUtils.isProxyCACertConfigured() + @@CustomProperties["ProxyCACertConfigured"] = "true" + end else @@CustomProperties["IsProxyConfigured"] = "false" isProxyConfigured = false diff --git a/source/plugins/ruby/CAdvisorMetricsAPIClient.rb b/source/plugins/ruby/CAdvisorMetricsAPIClient.rb index f3b1ccf57..017bfb08d 100644 --- a/source/plugins/ruby/CAdvisorMetricsAPIClient.rb +++ b/source/plugins/ruby/CAdvisorMetricsAPIClient.rb @@ -235,16 +235,17 @@ def getContainerCpuMetricItems(metricJSON, hostName, cpuMetricNameToCollect, met metricItem["ObjectName"] = Constants::OBJECT_NAME_K8S_CONTAINER metricItem["InstanceName"] = clusterId + "/" + podUid + "/" + containerName + metricCollection = {} metricCollection["CounterName"] = metricNametoReturn metricCollection["Value"] = metricValue metricItem["json_Collections"] = [] - metricCollections = [] - metricCollections.push(metricCollection) + metricCollections = [] + metricCollections.push(metricCollection) metricItem["json_Collections"] = metricCollections.to_json - metricItems.push(metricItem) - + metricItems.push(metricItem) + #Telemetry about agent performance begin # we can only do this much now. Ideally would like to use the docker image repository to find our pods/containers @@ -276,7 +277,7 @@ def getContainerCpuMetricItems(metricJSON, hostName, cpuMetricNameToCollect, met end #telemetry about containerlog Routing for daemonset telemetryProps["containerLogsRoute"] = @containerLogsRoute - + #telemetry about health model if (!@hmEnabled.nil? && !@hmEnabled.empty?) telemetryProps["hmEnabled"] = @hmEnabled @@ -525,13 +526,13 @@ def getContainerCpuMetricItemRate(metricJSON, hostName, cpuMetricNameToCollect, containerName = container["name"] metricValue = container["cpu"][cpuMetricNameToCollect] metricTime = metricPollTime #container["cpu"]["time"] - + metricItem = {} metricItem["Timestamp"] = metricTime metricItem["Host"] = hostName metricItem["ObjectName"] = Constants::OBJECT_NAME_K8S_CONTAINER metricItem["InstanceName"] = clusterId + "/" + podUid + "/" + containerName - + metricItem["json_Collections"] = [] metricCollection = {} metricCollection["CounterName"] = metricNametoReturn @@ -566,9 +567,9 @@ def getContainerCpuMetricItemRate(metricJSON, hostName, cpuMetricNameToCollect, end metricCollection["Value"] = metricValue - - metricCollections = [] - metricCollections.push(metricCollection) + + metricCollections = [] + metricCollections.push(metricCollection) metricItem["json_Collections"] = metricCollections.to_json metricItems.push(metricItem) #Telemetry about agent performance @@ -655,16 +656,16 @@ def getContainerMemoryMetricItems(metricJSON, hostName, memoryMetricNameToCollec metricItem["Host"] = hostName metricItem["ObjectName"] = Constants::OBJECT_NAME_K8S_CONTAINER metricItem["InstanceName"] = clusterId + "/" + podUid + "/" + containerName - + metricCollection = {} metricCollection["CounterName"] = metricNametoReturn metricCollection["Value"] = metricValue metricItem["json_Collections"] = [] - metricCollections = [] - metricCollections.push(metricCollection) + metricCollections = [] + metricCollections.push(metricCollection) metricItem["json_Collections"] = metricCollections.to_json - metricItems.push(metricItem) + metricItems.push(metricItem) #Telemetry about agent performance begin @@ -708,20 +709,21 @@ def getNodeMetricItem(metricJSON, hostName, metricCategory, metricNameToCollect, if !node[metricCategory].nil? metricValue = node[metricCategory][metricNameToCollect] metricTime = metricPollTime #node[metricCategory]["time"] - + metricItem["Timestamp"] = metricTime metricItem["Host"] = hostName metricItem["ObjectName"] = Constants::OBJECT_NAME_K8S_NODE metricItem["InstanceName"] = clusterId + "/" + nodeName + metricCollection = {} metricCollection["CounterName"] = metricNametoReturn metricCollection["Value"] = metricValue metricItem["json_Collections"] = [] - metricCollections = [] - metricCollections.push(metricCollection) - metricItem["json_Collections"] = metricCollections.to_json + metricCollections = [] + metricCollections.push(metricCollection) + metricItem["json_Collections"] = metricCollections.to_json end rescue => error @Log.warn("getNodeMetricItem failed: #{error} for metric #{metricNameToCollect}") @@ -824,19 +826,19 @@ def getNodeMetricItemRate(metricJSON, hostName, metricCategory, metricNameToColl end end end - + metricItem["Timestamp"] = metricTime metricItem["Host"] = hostName metricItem["ObjectName"] = Constants::OBJECT_NAME_K8S_NODE metricItem["InstanceName"] = clusterId + "/" + nodeName - + metricCollection = {} metricCollection["CounterName"] = metricNametoReturn metricCollection["Value"] = metricValue metricItem["json_Collections"] = [] - metricCollections = [] - metricCollections.push(metricCollection) + metricCollections = [] + metricCollections.push(metricCollection) metricItem["json_Collections"] = metricCollections.to_json end rescue => error @@ -859,19 +861,21 @@ def getNodeLastRebootTimeMetric(metricJSON, hostName, metricNametoReturn, metric metricValue = node["startTime"] metricTime = metricPollTime #Time.now.utc.iso8601 #2018-01-30T19:36:14Z + metricItem["Timestamp"] = metricTime metricItem["Host"] = hostName metricItem["ObjectName"] = Constants::OBJECT_NAME_K8S_NODE metricItem["InstanceName"] = clusterId + "/" + nodeName + metricCollection = {} metricCollection["CounterName"] = metricNametoReturn #Read it from /proc/uptime metricCollection["Value"] = DateTime.parse(metricTime).to_time.to_i - IO.read("/proc/uptime").split[0].to_f metricItem["json_Collections"] = [] - metricCollections = [] - metricCollections.push(metricCollection) + metricCollections = [] + metricCollections.push(metricCollection) metricItem["json_Collections"] = metricCollections.to_json rescue => error @Log.warn("getNodeLastRebootTimeMetric failed: #{error} ") @@ -900,14 +904,14 @@ def getContainerStartTimeMetricItems(metricJSON, hostName, metricNametoReturn, m metricItem["Host"] = hostName metricItem["ObjectName"] = Constants::OBJECT_NAME_K8S_CONTAINER metricItem["InstanceName"] = clusterId + "/" + podUid + "/" + containerName - + metricCollection = {} metricCollection["CounterName"] = metricNametoReturn metricCollection["Value"] = DateTime.parse(metricValue).to_time.to_i metricItem["json_Collections"] = [] - metricCollections = [] - metricCollections.push(metricCollection) + metricCollections = [] + metricCollections.push(metricCollection) metricItem["json_Collections"] = metricCollections.to_json metricItems.push(metricItem) end diff --git a/source/plugins/ruby/KubernetesApiClient.rb b/source/plugins/ruby/KubernetesApiClient.rb index 787744cee..8925248d7 100644 --- a/source/plugins/ruby/KubernetesApiClient.rb +++ b/source/plugins/ruby/KubernetesApiClient.rb @@ -88,7 +88,7 @@ def getTokenStr end end - def getClusterRegion(env = ENV) + def getClusterRegion(env=ENV) if env["AKS_REGION"] return env["AKS_REGION"] else @@ -97,7 +97,7 @@ def getClusterRegion(env = ENV) end end - def getResourceUri(resource, api_group, env = ENV) + def getResourceUri(resource, api_group, env=ENV) begin if env["KUBERNETES_SERVICE_HOST"] && env["KUBERNETES_PORT_443_TCP_PORT"] if api_group.nil? @@ -114,7 +114,7 @@ def getResourceUri(resource, api_group, env = ENV) end end - def getClusterName(env = ENV) + def getClusterName(env=ENV) return @@ClusterName if !@@ClusterName.nil? @@ClusterName = "None" begin @@ -148,7 +148,7 @@ def getClusterName(env = ENV) return @@ClusterName end - def getClusterId(env = ENV) + def getClusterId(env=ENV) return @@ClusterId if !@@ClusterId.nil? #By default initialize ClusterId to ClusterName. # In ACS/On-prem, we need to figure out how we can generate ClusterId @@ -456,19 +456,19 @@ def getContainerResourceRequestsAndLimits(pod, metricCategory, metricNameToColle metricCollection = {} metricCollection["CounterName"] = metricNametoReturn metricCollection["Value"] = metricValue - + metricProps["json_Collections"] = [] - metricCollections = [] - metricCollections.push(metricCollection) + metricCollections = [] + metricCollections.push(metricCollection) metricProps["json_Collections"] = metricCollections.to_json - metricItems.push(metricProps) + metricItems.push(metricProps) #No container level limit for the given metric, so default to node level limit else nodeMetricsHashKey = clusterId + "/" + nodeName + "_" + "allocatable" + "_" + metricNameToCollect if (metricCategory == "limits" && @@NodeMetrics.has_key?(nodeMetricsHashKey)) metricValue = @@NodeMetrics[nodeMetricsHashKey] #@Log.info("Limits not set for container #{clusterId + "/" + podUid + "/" + containerName} using node level limits: #{nodeMetricsHashKey}=#{metricValue} ") - + metricProps = {} metricProps["Timestamp"] = metricTime metricProps["Host"] = nodeName @@ -481,10 +481,10 @@ def getContainerResourceRequestsAndLimits(pod, metricCategory, metricNameToColle metricCollection["CounterName"] = metricNametoReturn metricCollection["Value"] = metricValue metricProps["json_Collections"] = [] - metricCollections = [] - metricCollections.push(metricCollection) + metricCollections = [] + metricCollections.push(metricCollection) metricProps["json_Collections"] = metricCollections.to_json - metricItems.push(metricProps) + metricItems.push(metricProps) end end end @@ -615,11 +615,11 @@ def parseNodeLimitsFromNodeItem(node, metricCategory, metricNameToCollect, metri metricCollection["CounterName"] = metricNametoReturn metricCollection["Value"] = metricValue metricCollections = [] - metricCollections.push(metricCollection) - + metricCollections.push(metricCollection) + metricItem["json_Collections"] = [] metricItem["json_Collections"] = metricCollections.to_json - + #push node level metrics to a inmem hash so that we can use it looking up at container level. #Currently if container level cpu & memory limits are not defined we default to node level limits @@NodeMetrics[clusterId + "/" + node["metadata"]["name"] + "_" + metricCategory + "_" + metricNameToCollect] = metricValue @@ -778,7 +778,7 @@ def getResourcesAndContinuationToken(uri, api_group: nil) return continuationToken, resourceInventory end #getResourcesAndContinuationToken - def getKubeAPIServerUrl(env = ENV) + def getKubeAPIServerUrl(env=ENV) apiServerUrl = nil begin if env["KUBERNETES_SERVICE_HOST"] && env["KUBERNETES_PORT_443_TCP_PORT"] diff --git a/source/plugins/ruby/filter_cadvisor2mdm.rb b/source/plugins/ruby/filter_cadvisor2mdm.rb index 32b3a3884..6bafa372a 100644 --- a/source/plugins/ruby/filter_cadvisor2mdm.rb +++ b/source/plugins/ruby/filter_cadvisor2mdm.rb @@ -160,17 +160,17 @@ def filter(tag, time, record) begin if @process_incoming_stream - # Check if insights metrics for PV metrics + # Check if insights metrics for PV metrics if record["Name"] == Constants::PV_USED_BYTES return filterPVInsightsMetrics(record) end object_name = record["ObjectName"] counter_name = JSON.parse(record["json_Collections"])[0]["CounterName"] - + percentage_metric_value = 0.0 allocatable_percentage_metric_value = 0.0 - metric_value = JSON.parse(record["json_Collections"])[0]["Value"] + metric_value = JSON.parse(record["json_Collections"])[0]["Value"] if object_name == Constants::OBJECT_NAME_K8S_NODE && @metrics_to_collect_hash.key?(counter_name.downcase) # Compute and send % CPU and Memory @@ -215,7 +215,7 @@ def filter(tag, time, record) else allocatable_percentage_metric_value = 0.0 end - end + end @log.info "percentage_metric_value for metric: #{metric_name} for instance: #{record["Host"]} percentage: #{percentage_metric_value} allocatable_percentage: #{allocatable_percentage_metric_value}" # do some sanity checking. @@ -292,7 +292,7 @@ def filter(tag, time, record) end end - def filterPVInsightsMetrics(record) + def filterPVInsightsMetrics(record) begin mdmMetrics = [] if record["Name"] == Constants::PV_USED_BYTES && @metrics_to_collect_hash.key?(record["Name"].downcase) @@ -356,8 +356,8 @@ def ensure_cpu_memory_capacity_and_allocatable_set end if !nodeInventory.nil? cpu_capacity_json = KubernetesApiClient.parseNodeLimits(nodeInventory, "capacity", "cpu", "cpuCapacityNanoCores") - if !cpu_capacity_json.nil? - metricVal = JSON.parse(cpu_capacity_json[0]["json_Collections"])[0]["Value"] + if !cpu_capacity_json.nil? + metricVal = JSON.parse(cpu_capacity_json[0]["json_Collections"])[0]["Value"] if !metricVal.to_s.nil? @cpu_capacity = metricVal @log.info "CPU Limit #{@cpu_capacity}" @@ -366,8 +366,8 @@ def ensure_cpu_memory_capacity_and_allocatable_set @log.info "Error getting cpu_capacity" end memory_capacity_json = KubernetesApiClient.parseNodeLimits(nodeInventory, "capacity", "memory", "memoryCapacityBytes") - if !memory_capacity_json.nil? - metricVal = JSON.parse(cpu_capacity_json[0]["json_Collections"])[0]["Value"] + if !memory_capacity_json.nil? + metricVal = JSON.parse(cpu_capacity_json[0]["json_Collections"])[0]["Value"] if !metricVal.to_s.nil? @memory_capacity = metricVal @log.info "Memory Limit #{@memory_capacity}" diff --git a/source/plugins/ruby/filter_health_model_builder.rb b/source/plugins/ruby/filter_health_model_builder.rb index 3262935aa..9decda881 100644 --- a/source/plugins/ruby/filter_health_model_builder.rb +++ b/source/plugins/ruby/filter_health_model_builder.rb @@ -2,276 +2,285 @@ # frozen_string_literal: true -require "fluent/plugin/filter" +require 'fluent/plugin/filter' module Fluent::Plugin - require_relative "extension_utils" - require "logger" - require "yajl/json_gem" - Dir[File.join(__dir__, "./health", "*.rb")].each { |file| require file } - - class FilterHealthModelBuilder < Filter - include HealthModel - Fluent::Plugin.register_filter("health_model_builder", self) - - config_param :enable_log, :integer, :default => 0 - config_param :log_path, :string, :default => "/var/opt/microsoft/docker-cimprov/log/filter_health_model_builder.log" - config_param :model_definition_path, :default => "/etc/opt/microsoft/docker-cimprov/health/health_model_definition.json" - config_param :health_monitor_config_path, :default => "/etc/opt/microsoft/docker-cimprov/health/healthmonitorconfig.json" - config_param :health_state_serialized_path, :default => "/mnt/azure/health_model_state.json" - attr_reader :buffer, :model_builder, :health_model_definition, :monitor_factory, :state_finalizers, :monitor_set, :model_builder, :hierarchy_builder, :resources, :kube_api_down_handler, :provider, :reducer, :state, :generator, :telemetry - - @@cluster_id = KubernetesApiClient.getClusterId - @@token_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" - @@cert_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - @@cluster_health_model_enabled = HealthMonitorUtils.is_cluster_health_model_enabled - - def initialize - begin - super - @rewrite_tag = "oneagent.containerInsights.KUBE_HEALTH_BLOB" - @buffer = HealthModel::HealthModelBuffer.new - @cluster_health_state = ClusterHealthState.new(@@token_file_path, @@cert_file_path) - @health_model_definition = HealthModel::ParentMonitorProvider.new(HealthModel::HealthModelDefinitionParser.new(@model_definition_path).parse_file) - @monitor_factory = HealthModel::MonitorFactory.new - @hierarchy_builder = HealthHierarchyBuilder.new(@health_model_definition, @monitor_factory) - # TODO: Figure out if we need to add NodeMonitorHierarchyReducer to the list of finalizers. For now, dont compress/optimize, since it becomes impossible to construct the model on the UX side - @state_finalizers = [HealthModel::AggregateMonitorStateFinalizer.new] - @monitor_set = HealthModel::MonitorSet.new - @model_builder = HealthModel::HealthModelBuilder.new(@hierarchy_builder, @state_finalizers, @monitor_set) - @kube_api_down_handler = HealthKubeApiDownHandler.new - @resources = HealthKubernetesResources.instance - @reducer = HealthSignalReducer.new - @generator = HealthMissingSignalGenerator.new - @provider = HealthMonitorProvider.new(@@cluster_id, HealthMonitorUtils.get_cluster_labels, @resources, @health_monitor_config_path) - @cluster_old_state = "none" - @cluster_new_state = "none" - @container_cpu_memory_records = [] - @telemetry = HealthMonitorTelemetry.new - @state = HealthMonitorState.new - # move network calls to the end. This will ensure all the instance variables get initialized - if @@cluster_health_model_enabled - deserialized_state_info = @cluster_health_state.get_state - @state.initialize_state(deserialized_state_info) - end - rescue => e - ApplicationInsightsUtility.sendExceptionTelemetry(e, { "FeatureArea" => "Health" }) - end - end + require_relative 'extension_utils' + require 'logger' + require 'yajl/json_gem' + Dir[File.join(__dir__, './health', '*.rb')].each { |file| require file } - def configure(conf) - begin - super - @log = nil - if @enable_log - @log = Logger.new(@log_path, "weekly") - @log.info "Starting filter_health_model_builder plugin" - end - rescue => e - ApplicationInsightsUtility.sendExceptionTelemetry(e, { "FeatureArea" => "Health" }) - end - end - def start - super - end + class FilterHealthModelBuilder < Filter + include HealthModel + Fluent::Plugin.register_filter('health_model_builder', self) - def shutdown - super - end + config_param :enable_log, :integer, :default => 0 + config_param :log_path, :string, :default => '/var/opt/microsoft/docker-cimprov/log/filter_health_model_builder.log' + config_param :model_definition_path, :default => '/etc/opt/microsoft/docker-cimprov/health/health_model_definition.json' + config_param :health_monitor_config_path, :default => '/etc/opt/microsoft/docker-cimprov/health/healthmonitorconfig.json' + config_param :health_state_serialized_path, :default => '/mnt/azure/health_model_state.json' + attr_reader :buffer, :model_builder, :health_model_definition, :monitor_factory, :state_finalizers, :monitor_set, :model_builder, :hierarchy_builder, :resources, :kube_api_down_handler, :provider, :reducer, :state, :generator, :telemetry + + + + @@cluster_id = KubernetesApiClient.getClusterId + @@token_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/token" + @@cert_file_path = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + @@cluster_health_model_enabled = HealthMonitorUtils.is_cluster_health_model_enabled + + def initialize + begin + super + @rewrite_tag = 'oneagent.containerInsights.KUBE_HEALTH_BLOB' + @buffer = HealthModel::HealthModelBuffer.new + @cluster_health_state = ClusterHealthState.new(@@token_file_path, @@cert_file_path) + @health_model_definition = HealthModel::ParentMonitorProvider.new(HealthModel::HealthModelDefinitionParser.new(@model_definition_path).parse_file) + @monitor_factory = HealthModel::MonitorFactory.new + @hierarchy_builder = HealthHierarchyBuilder.new(@health_model_definition, @monitor_factory) + # TODO: Figure out if we need to add NodeMonitorHierarchyReducer to the list of finalizers. For now, dont compress/optimize, since it becomes impossible to construct the model on the UX side + @state_finalizers = [HealthModel::AggregateMonitorStateFinalizer.new] + @monitor_set = HealthModel::MonitorSet.new + @model_builder = HealthModel::HealthModelBuilder.new(@hierarchy_builder, @state_finalizers, @monitor_set) + @kube_api_down_handler = HealthKubeApiDownHandler.new + @resources = HealthKubernetesResources.instance + @reducer = HealthSignalReducer.new + @generator = HealthMissingSignalGenerator.new + @provider = HealthMonitorProvider.new(@@cluster_id, HealthMonitorUtils.get_cluster_labels, @resources, @health_monitor_config_path) + @cluster_old_state = 'none' + @cluster_new_state = 'none' + @container_cpu_memory_records = [] + @telemetry = HealthMonitorTelemetry.new + @state = HealthMonitorState.new + # move network calls to the end. This will ensure all the instance variables get initialized + if @@cluster_health_model_enabled + deserialized_state_info = @cluster_health_state.get_state + @state.initialize_state(deserialized_state_info) + end - def filter_stream(tag, es) - if !@@cluster_health_model_enabled - @log.info "Cluster Health Model disabled in filter_health_model_builder" - return Fluent::MultiEventStream.new - end - begin - new_es = Fluent::MultiEventStream.new - time = Time.now - if ExtensionUtils.isAADMSIAuthMode() - $log.info("filter_health_model_builder::enumerate: AAD AUTH MSI MODE") - if @rewrite_tag.nil? || !@rewrite_tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) - @rewrite_tag = ExtensionUtils.getOutputStreamId(Constants::KUBE_HEALTH_DATA_TYPE) - end - $log.info("filter_health_model_builder::filter_stream: using tag -#{@rewrite_tag} @ #{Time.now.utc.iso8601}") + rescue => e + ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) + end end - if tag.start_with?("kubehealth.DaemonSet.Node") - node_records = [] - if !es.nil? - es.each { |time, record| - node_records.push(record) - } - @buffer.add_to_buffer(node_records) - end - return Fluent::MultiEventStream.new - elsif tag.start_with?("kubehealth.DaemonSet.Container") - container_records = [] - if !es.nil? - es.each { |time, record| - container_records.push(record) - } - end - container_records_aggregator = HealthContainerCpuMemoryAggregator.new(@resources, @provider) - if @container_cpu_memory_records.nil? - @log.info "@container_cpu_memory_records was not initialized" - @container_cpu_memory_records = [] #in some clusters, this is null, so initialize it again. - end - @container_cpu_memory_records.push(*container_records) # push the records for aggregation later - return Fluent::MultiEventStream.new - elsif tag.start_with?("kubehealth.ReplicaSet") - records = [] - es.each { |time, record| - records.push(record) - } - @buffer.add_to_buffer(records) # in_kube_health records - - aggregated_container_records = [] - if !@container_cpu_memory_records.nil? && !@container_cpu_memory_records.empty? - container_records_aggregator = HealthContainerCpuMemoryAggregator.new(@resources, @provider) - deduped_records = container_records_aggregator.dedupe_records(@container_cpu_memory_records) - container_records_aggregator.aggregate(deduped_records) - container_records_aggregator.compute_state - aggregated_container_records = container_records_aggregator.get_records - end - @buffer.add_to_buffer(aggregated_container_records) #container cpu/memory records - records_to_process = @buffer.get_buffer - @buffer.reset_buffer - @container_cpu_memory_records = [] - - health_monitor_records = [] - records_to_process.each do |record| - monitor_instance_id = record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] - monitor_id = record[HealthMonitorRecordFields::MONITOR_ID] - #HealthMonitorRecord - health_monitor_record = HealthMonitorRecord.new( - record[HealthMonitorRecordFields::MONITOR_ID], - record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID], - record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED], - record[HealthMonitorRecordFields::DETAILS]["state"], - @provider.get_labels(record), - @provider.get_config(monitor_id), - record[HealthMonitorRecordFields::DETAILS] - ) - health_monitor_records.push(health_monitor_record) - #puts "#{monitor_instance_id} #{instance_state.new_state} #{instance_state.old_state} #{instance_state.should_send}" - end - - @log.info "health_monitor_records.size #{health_monitor_records.size}" - # Dedupe daemonset signals - # Remove unit monitor signals for “gone” objects - # update state for the reduced set of signals - reduced_records = @reducer.reduce_signals(health_monitor_records, @resources) - reduced_records.each { |record| - @state.update_state(record, - @provider.get_config(record.monitor_id), - false, - @telemetry) - # get the health state based on the monitor's operational state - # update state calls updates the state of the monitor based on configuration and history of the the monitor records - record.state = @state.get_state(record.monitor_instance_id).new_state - } - @log.info "after deduping and removing gone objects reduced_records.size #{reduced_records.size}" - - reduced_records = @kube_api_down_handler.handle_kube_api_down(reduced_records) - @log.info "after kube api down handler health_monitor_records.size #{health_monitor_records.size}" - - #get the list of 'none' and 'unknown' signals - missing_signals = @generator.get_missing_signals(@@cluster_id, reduced_records, @resources, @provider) - - @log.info "after getting missing signals missing_signals.size #{missing_signals.size}" - #update state for missing signals - missing_signals.each { |signal| - @state.update_state(signal, @provider.get_config(signal.monitor_id), false, @telemetry) - @log.info "After Updating #{@state.get_state(signal.monitor_instance_id)} #{@state.get_state(signal.monitor_instance_id).new_state}" - # for unknown/none records, update the "monitor state" to be the latest state (new_state) of the monitor instance from the state - signal.state = @state.get_state(signal.monitor_instance_id).new_state - } - - @generator.update_last_received_records(reduced_records) - all_records = reduced_records.clone - all_records.push(*missing_signals) - - @log.info "after Adding missing signals all_records.size #{all_records.size}" - - HealthMonitorHelpers.add_agentpool_node_label_if_not_present(all_records) - - # build the health model - @model_builder.process_records(all_records) - all_monitors = @model_builder.finalize_model - - @log.info "after building health_model #{all_monitors.size}" - - # update the state for aggregate monitors (unit monitors are updated above) - all_monitors.each { |monitor_instance_id, monitor| - if monitor.is_aggregate_monitor - @state.update_state(monitor, - @provider.get_config(monitor.monitor_id), - true, - @telemetry) + def configure(conf) + begin + super + @log = nil + if @enable_log + @log = Logger.new(@log_path, 'weekly') + @log.info 'Starting filter_health_model_builder plugin' + end + rescue => e + ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) end + end + + def start + super + end - instance_state = @state.get_state(monitor_instance_id) - #puts "#{monitor_instance_id} #{instance_state.new_state} #{instance_state.old_state} #{instance_state.should_send}" - should_send = instance_state.should_send + def shutdown + super + end - # always send cluster monitor as a heartbeat - if !should_send && monitor_instance_id != MonitorId::CLUSTER - all_monitors.delete(monitor_instance_id) + def filter_stream(tag, es) + if !@@cluster_health_model_enabled + @log.info "Cluster Health Model disabled in filter_health_model_builder" + return Fluent::MultiEventStream.new end - } - - @log.info "after optimizing health signals all_monitors.size #{all_monitors.size}" - - # for each key in monitor.keys, - # get the state from health_monitor_state - # generate the record to send - emit_time = Fluent::Engine.now - all_monitors.keys.each { |key| - record = @provider.get_record(all_monitors[key], state) - if record[HealthMonitorRecordFields::MONITOR_ID] == MonitorId::CLUSTER - if !record[HealthMonitorRecordFields::DETAILS].nil? - details = JSON.parse(record[HealthMonitorRecordFields::DETAILS]) - details[HealthMonitorRecordFields::HEALTH_MODEL_DEFINITION_VERSION] = "#{ENV["HEALTH_MODEL_DEFINITION_VERSION"]}" - record[HealthMonitorRecordFields::DETAILS] = details.to_json - end - if all_monitors.size > 1 - old_state = record[HealthMonitorRecordFields::OLD_STATE] - new_state = record[HealthMonitorRecordFields::NEW_STATE] - if old_state != new_state && @cluster_old_state != old_state && @cluster_new_state != new_state - ApplicationInsightsUtility.sendCustomEvent("HealthModel_ClusterStateChanged", { "old_state" => old_state, "new_state" => new_state, "monitor_count" => all_monitors.size }) - @log.info "sent telemetry for cluster state change from #{record["OldState"]} to #{record["NewState"]}" - @cluster_old_state = old_state - @cluster_new_state = new_state + begin + new_es = Fluent::MultiEventStream.new + time = Time.now + if ExtensionUtils.isAADMSIAuthMode() + $log.info("filter_health_model_builder::enumerate: AAD AUTH MSI MODE") + if @rewrite_tag.nil? || !@rewrite_tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @rewrite_tag = ExtensionUtils.getOutputStreamId(Constants::KUBE_HEALTH_DATA_TYPE) + end + $log.info("filter_health_model_builder::filter_stream: using tag -#{@rewrite_tag} @ #{Time.now.utc.iso8601}") end - end + + if tag.start_with?("kubehealth.DaemonSet.Node") + node_records = [] + if !es.nil? + es.each{|time, record| + node_records.push(record) + } + @buffer.add_to_buffer(node_records) + end + return Fluent::MultiEventStream.new + elsif tag.start_with?("kubehealth.DaemonSet.Container") + container_records = [] + if !es.nil? + es.each{|time, record| + container_records.push(record) + } + end + container_records_aggregator = HealthContainerCpuMemoryAggregator.new(@resources, @provider) + if @container_cpu_memory_records.nil? + @log.info "@container_cpu_memory_records was not initialized" + @container_cpu_memory_records = [] #in some clusters, this is null, so initialize it again. + end + @container_cpu_memory_records.push(*container_records) # push the records for aggregation later + return Fluent::MultiEventStream.new + elsif tag.start_with?("kubehealth.ReplicaSet") + records = [] + es.each{|time, record| + records.push(record) + } + @buffer.add_to_buffer(records) # in_kube_health records + + aggregated_container_records = [] + if !@container_cpu_memory_records.nil? && !@container_cpu_memory_records.empty? + container_records_aggregator = HealthContainerCpuMemoryAggregator.new(@resources, @provider) + deduped_records = container_records_aggregator.dedupe_records(@container_cpu_memory_records) + container_records_aggregator.aggregate(deduped_records) + container_records_aggregator.compute_state + aggregated_container_records = container_records_aggregator.get_records + end + @buffer.add_to_buffer(aggregated_container_records) #container cpu/memory records + records_to_process = @buffer.get_buffer + @buffer.reset_buffer + @container_cpu_memory_records = [] + + health_monitor_records = [] + records_to_process.each do |record| + monitor_instance_id = record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID] + monitor_id = record[HealthMonitorRecordFields::MONITOR_ID] + #HealthMonitorRecord + health_monitor_record = HealthMonitorRecord.new( + record[HealthMonitorRecordFields::MONITOR_ID], + record[HealthMonitorRecordFields::MONITOR_INSTANCE_ID], + record[HealthMonitorRecordFields::TIME_FIRST_OBSERVED], + record[HealthMonitorRecordFields::DETAILS]["state"], + @provider.get_labels(record), + @provider.get_config(monitor_id), + record[HealthMonitorRecordFields::DETAILS] + ) + health_monitor_records.push(health_monitor_record) + #puts "#{monitor_instance_id} #{instance_state.new_state} #{instance_state.old_state} #{instance_state.should_send}" + end + + @log.info "health_monitor_records.size #{health_monitor_records.size}" + # Dedupe daemonset signals + # Remove unit monitor signals for “gone” objects + # update state for the reduced set of signals + reduced_records = @reducer.reduce_signals(health_monitor_records, @resources) + reduced_records.each{|record| + @state.update_state(record, + @provider.get_config(record.monitor_id), + false, + @telemetry + ) + # get the health state based on the monitor's operational state + # update state calls updates the state of the monitor based on configuration and history of the the monitor records + record.state = @state.get_state(record.monitor_instance_id).new_state + } + @log.info "after deduping and removing gone objects reduced_records.size #{reduced_records.size}" + + reduced_records = @kube_api_down_handler.handle_kube_api_down(reduced_records) + @log.info "after kube api down handler health_monitor_records.size #{health_monitor_records.size}" + + #get the list of 'none' and 'unknown' signals + missing_signals = @generator.get_missing_signals(@@cluster_id, reduced_records, @resources, @provider) + + @log.info "after getting missing signals missing_signals.size #{missing_signals.size}" + #update state for missing signals + missing_signals.each{|signal| + + @state.update_state(signal, @provider.get_config(signal.monitor_id), false, @telemetry) + @log.info "After Updating #{@state.get_state(signal.monitor_instance_id)} #{@state.get_state(signal.monitor_instance_id).new_state}" + # for unknown/none records, update the "monitor state" to be the latest state (new_state) of the monitor instance from the state + signal.state = @state.get_state(signal.monitor_instance_id).new_state + } + + @generator.update_last_received_records(reduced_records) + all_records = reduced_records.clone + all_records.push(*missing_signals) + + @log.info "after Adding missing signals all_records.size #{all_records.size}" + + HealthMonitorHelpers.add_agentpool_node_label_if_not_present(all_records) + + # build the health model + @model_builder.process_records(all_records) + all_monitors = @model_builder.finalize_model + + @log.info "after building health_model #{all_monitors.size}" + + # update the state for aggregate monitors (unit monitors are updated above) + all_monitors.each{|monitor_instance_id, monitor| + if monitor.is_aggregate_monitor + @state.update_state(monitor, + @provider.get_config(monitor.monitor_id), + true, + @telemetry + ) + end + + instance_state = @state.get_state(monitor_instance_id) + #puts "#{monitor_instance_id} #{instance_state.new_state} #{instance_state.old_state} #{instance_state.should_send}" + should_send = instance_state.should_send + + # always send cluster monitor as a heartbeat + if !should_send && monitor_instance_id != MonitorId::CLUSTER + all_monitors.delete(monitor_instance_id) + end + } + + @log.info "after optimizing health signals all_monitors.size #{all_monitors.size}" + + + # for each key in monitor.keys, + # get the state from health_monitor_state + # generate the record to send + emit_time = Fluent::Engine.now + all_monitors.keys.each{|key| + record = @provider.get_record(all_monitors[key], state) + if record[HealthMonitorRecordFields::MONITOR_ID] == MonitorId::CLUSTER + if !record[HealthMonitorRecordFields::DETAILS].nil? + details = JSON.parse(record[HealthMonitorRecordFields::DETAILS]) + details[HealthMonitorRecordFields::HEALTH_MODEL_DEFINITION_VERSION] = "#{ENV['HEALTH_MODEL_DEFINITION_VERSION']}" + record[HealthMonitorRecordFields::DETAILS] = details.to_json + end + if all_monitors.size > 1 + old_state = record[HealthMonitorRecordFields::OLD_STATE] + new_state = record[HealthMonitorRecordFields::NEW_STATE] + if old_state != new_state && @cluster_old_state != old_state && @cluster_new_state != new_state + ApplicationInsightsUtility.sendCustomEvent("HealthModel_ClusterStateChanged",{"old_state" => old_state , "new_state" => new_state, "monitor_count" => all_monitors.size}) + @log.info "sent telemetry for cluster state change from #{record['OldState']} to #{record['NewState']}" + @cluster_old_state = old_state + @cluster_new_state = new_state + end + end + end + new_es.add(emit_time, record) + } + + #emit the stream + router.emit_stream(@rewrite_tag, new_es) + + #initialize monitor_set and model_builder + @monitor_set = HealthModel::MonitorSet.new + @model_builder = HealthModel::HealthModelBuilder.new(@hierarchy_builder, @state_finalizers, @monitor_set) + + #update cluster state custom resource + @cluster_health_state.update_state(@state.to_h) + @telemetry.send + # return an empty event stream, else the match will throw a NoMethodError + return Fluent::MultiEventStream.new + elsif tag.start_with?(@rewrite_tag) + # this filter also acts as a pass through as we are rewriting the tag and emitting to the fluent stream + es + else + raise "Invalid tag #{tag} received" + end + + rescue => e + ApplicationInsightsUtility.sendExceptionTelemetry(e, {"FeatureArea" => "Health"}) + @log.warn "Message: #{e.message} Backtrace: #{e.backtrace}" + return nil end - new_es.add(emit_time, record) - } - - #emit the stream - router.emit_stream(@rewrite_tag, new_es) - - #initialize monitor_set and model_builder - @monitor_set = HealthModel::MonitorSet.new - @model_builder = HealthModel::HealthModelBuilder.new(@hierarchy_builder, @state_finalizers, @monitor_set) - - #update cluster state custom resource - @cluster_health_state.update_state(@state.to_h) - @telemetry.send - # return an empty event stream, else the match will throw a NoMethodError - return Fluent::MultiEventStream.new - elsif tag.start_with?(@rewrite_tag) - # this filter also acts as a pass through as we are rewriting the tag and emitting to the fluent stream - es - else - raise "Invalid tag #{tag} received" end - rescue => e - ApplicationInsightsUtility.sendExceptionTelemetry(e, { "FeatureArea" => "Health" }) - @log.warn "Message: #{e.message} Backtrace: #{e.backtrace}" - return nil - end end - end end diff --git a/source/plugins/ruby/in_kube_nodes.rb b/source/plugins/ruby/in_kube_nodes.rb index fa6477856..5a52a089b 100644 --- a/source/plugins/ruby/in_kube_nodes.rb +++ b/source/plugins/ruby/in_kube_nodes.rb @@ -145,31 +145,6 @@ def enumerate end nodesAPIChunkStartTime = (Time.now.to_f * 1000).to_i - @nodesAPIE2ELatencyMs = 0 - @nodeInventoryE2EProcessingLatencyMs = 0 - nodeInventoryStartTime = (Time.now.to_f * 1000).to_i - - if @extensionUtils.isAADMSIAuthMode() - $log.info("in_kube_nodes::enumerate: AAD AUTH MSI MODE") - if @kubeperfTag.nil? || !@kubeperfTag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) - @kubeperfTag = @extensionUtils.getOutputStreamId(Constants::PERF_DATA_TYPE) - end - if @insightsMetricsTag.nil? || !@insightsMetricsTag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) - @insightsMetricsTag = @extensionUtils.getOutputStreamId(Constants::INSIGHTS_METRICS_DATA_TYPE) - end - if @ContainerNodeInventoryTag.nil? || !@ContainerNodeInventoryTag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) - @ContainerNodeInventoryTag = @extensionUtils.getOutputStreamId(Constants::CONTAINER_NODE_INVENTORY_DATA_TYPE) - end - if @tag.nil? || !@tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) - @tag = @extensionUtils.getOutputStreamId(Constants::KUBE_NODE_INVENTORY_DATA_TYPE) - end - $log.info("in_kube_nodes::enumerate: using perf tag -#{@kubeperfTag} @ #{Time.now.utc.iso8601}") - $log.info("in_kube_nodes::enumerate: using insightsmetrics tag -#{@insightsMetricsTag} @ #{Time.now.utc.iso8601}") - $log.info("in_kube_nodes::enumerate: using containernodeinventory tag -#{@ContainerNodeInventoryTag} @ #{Time.now.utc.iso8601}") - $log.info("in_kube_nodes::enumerate: using kubenodeinventory tag -#{@tag} @ #{Time.now.utc.iso8601}") - end - nodesAPIChunkStartTime = (Time.now.to_f * 1000).to_i - # Initializing continuation token to nil continuationToken = nil $log.info("in_kube_nodes::enumerate : Getting nodes from Kube API @ #{Time.now.utc.iso8601}") @@ -585,6 +560,9 @@ def getNodeTelemetryProps(item) properties["OperatingSystem"] = nodeInfo["operatingSystem"] properties["KernelVersion"] = nodeInfo["kernelVersion"] properties["OSImage"] = nodeInfo["osImage"] + if nodeInfo["architecture"] == "arm64" + properties["Architecture"] = nodeInfo["architecture"] + end containerRuntimeVersion = nodeInfo["containerRuntimeVersion"] if containerRuntimeVersion.downcase.start_with?("docker://") properties["DockerVersion"] = containerRuntimeVersion.split("//")[1] diff --git a/source/plugins/ruby/in_kube_pvinventory.rb b/source/plugins/ruby/in_kube_pvinventory.rb index b548bd5c2..fccfd459d 100644 --- a/source/plugins/ruby/in_kube_pvinventory.rb +++ b/source/plugins/ruby/in_kube_pvinventory.rb @@ -1,7 +1,7 @@ #!/usr/local/bin/ruby # frozen_string_literal: true -require "fluent/plugin/input" +require 'fluent/plugin/input' module Fluent::Plugin class Kube_PVInventory_Input < Input @@ -100,6 +100,7 @@ def enumerate if (timeDifferenceInMinutes >= Constants::TELEMETRY_FLUSH_INTERVAL_IN_MINUTES) telemetryFlush = true end + # Flush AppInsights telemetry once all the processing is done if telemetryFlush == true telemetryProperties = {} @@ -107,6 +108,7 @@ def enumerate ApplicationInsightsUtility.sendCustomEvent(Constants::PV_INVENTORY_HEART_BEAT_EVENT, telemetryProperties) @@pvTelemetryTimeTracker = DateTime.now.to_time.to_i end + rescue => errorStr $log.warn "in_kube_pvinventory::enumerate:Failed in enumerate: #{errorStr}" $log.debug_backtrace(errorStr.backtrace) @@ -130,7 +132,7 @@ def parse_and_emit_records(pvInventory, batchTime = Time.utc.iso8601) record["ClusterName"] = KubernetesApiClient.getClusterName record["PVName"] = item["metadata"]["name"] record["PVStatus"] = item["status"]["phase"] - record["PVAccessModes"] = item["spec"]["accessModes"].join(", ") + record["PVAccessModes"] = item["spec"]["accessModes"].join(', ') record["PVStorageClassName"] = item["spec"]["storageClassName"] record["PVCapacityBytes"] = KubernetesApiClient.getMetricNumericValue("memory", item["spec"]["capacity"]["storage"]) record["PVCreationTimeStamp"] = item["metadata"]["creationTimestamp"] @@ -166,6 +168,7 @@ def parse_and_emit_records(pvInventory, batchTime = Time.utc.iso8601) if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp("true") == 0) $log.info("kubePVInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") end + rescue => errorStr $log.warn "Failed in parse_and_emit_record for in_kube_pvinventory: #{errorStr}" $log.debug_backtrace(errorStr.backtrace) @@ -195,6 +198,7 @@ def getTypeInfo(item) begin if !item["spec"].nil? (Constants::PV_TYPES).each do |pvType| + # PV is this type if !item["spec"][pvType].nil? @@ -210,6 +214,7 @@ def getTypeInfo(item) # Can only have one type: return right away when found return pvType, typeInfo + end end end @@ -223,6 +228,7 @@ def getTypeInfo(item) return nil, {} end + def run_periodic @mutex.lock done = @finished diff --git a/source/plugins/ruby/kubelet_utils.rb b/source/plugins/ruby/kubelet_utils.rb index c9114d9e8..368ca8639 100644 --- a/source/plugins/ruby/kubelet_utils.rb +++ b/source/plugins/ruby/kubelet_utils.rb @@ -52,7 +52,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) cpu_allocatable = 1.0 memory_allocatable = 1.0 - + allocatable_response = CAdvisorMetricsAPIClient.getCongifzCAdvisor(winNode: nil) parsed_response = JSON.parse(allocatable_response.body) @@ -66,7 +66,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::kubereserved_cpu: #{errorStr}" kubereserved_cpu = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::kubereserved_cpu: #{errorStr}") - end + end begin kubereserved_memory = parsed_response["kubeletconfig"]["kubeReserved"]["memory"] @@ -78,7 +78,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::kubereserved_memory: #{errorStr}" kubereserved_memory = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::kubereserved_memory: #{errorStr}") - end + end begin systemReserved_cpu = parsed_response["kubeletconfig"]["systemReserved"]["cpu"] if systemReserved_cpu.nil? || systemReserved_cpu == "" @@ -90,7 +90,7 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::systemReserved_cpu: #{errorStr}" systemReserved_cpu = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::systemReserved_cpu: #{errorStr}") - end + end begin explicitlyReserved_cpu = parsed_response["kubeletconfig"]["reservedCPUs"] @@ -103,19 +103,19 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::explicitlyReserved_cpu: #{errorStr}" explicitlyReserved_cpu = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::explicitlyReserved_cpu: #{errorStr}") - end + end begin - systemReserved_memory = parsed_response["kubeletconfig"]["systemReserved"]["memory"] - if systemReserved_memory.nil? || systemReserved_memory == "" + systemReserved_memory = parsed_response["kubeletconfig"]["systemReserved"]["memory"] + if systemReserved_memory.nil? || systemReserved_memory == "" systemReserved_memory = "0.0" - end - @log.info "get_node_allocatable::systemReserved_memory #{systemReserved_memory}" + end + @log.info "get_node_allocatable::systemReserved_memory #{systemReserved_memory}" rescue => errorStr - @log.error "Error in get_node_allocatable::systemReserved_memory: #{errorStr}" - systemReserved_memory = "0.0" + @log.error "Error in get_node_allocatable::systemReserved_memory: #{errorStr}" + systemReserved_memory = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::systemReserved_memory: #{errorStr}") - end + end begin evictionHard_memory = parsed_response["kubeletconfig"]["evictionHard"]["memory.available"] @@ -127,16 +127,16 @@ def get_node_allocatable(cpu_capacity, memory_capacity) @log.error "Error in get_node_allocatable::evictionHard_memory: #{errorStr}" evictionHard_memory = "0.0" ApplicationInsightsUtility.sendExceptionTelemetry("Error in get_node_allocatable::evictionHard_memory: #{errorStr}") - end + end # do calculation in nanocore since that's what KubernetesApiClient.getMetricNumericValue expects cpu_capacity_number = cpu_capacity.to_i * 1000.0 ** 2 # subtract to get allocatable. Formula : Allocatable = Capacity - ( kube reserved + system reserved + eviction threshold ) # https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#node-allocatable if KubernetesApiClient.getMetricNumericValue("cpu", explicitlyReserved_cpu) > 0 - cpu_allocatable = cpu_capacity_number - KubernetesApiClient.getMetricNumericValue("cpu", explicitlyReserved_cpu) + cpu_allocatable = cpu_capacity_number - KubernetesApiClient.getMetricNumericValue("cpu", explicitlyReserved_cpu) else - cpu_allocatable = cpu_capacity_number - (KubernetesApiClient.getMetricNumericValue("cpu", kubereserved_cpu) + KubernetesApiClient.getMetricNumericValue("cpu", systemReserved_cpu)) + cpu_allocatable = cpu_capacity_number - (KubernetesApiClient.getMetricNumericValue("cpu", kubereserved_cpu) + KubernetesApiClient.getMetricNumericValue("cpu", systemReserved_cpu)) end # convert back to units similar to what we get for capacity cpu_allocatable = cpu_allocatable / (1000.0 ** 2) diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index 82d6e07db..e10a2049f 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -96,17 +96,14 @@ def start end @@post_request_url = @@post_request_url_template % { aks_region: aks_region, aks_resource_id: aks_resource_id } @post_request_uri = URI.parse(@@post_request_url) - if (!!@isArcK8sCluster) - proxy = (ProxyUtils.getProxyConfiguration) - if proxy.nil? || proxy.empty? - @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port) - else - @log.info "Proxy configured on this cluster: #{aks_resource_id}" - @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port, proxy[:addr], proxy[:port], proxy[:user], proxy[:pass]) - end - else + proxy = (ProxyUtils.getProxyConfiguration) + if proxy.nil? || proxy.empty? @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port) + else + @log.info "Proxy configured on this cluster: #{aks_resource_id}" + @http_client = Net::HTTP.new(@post_request_uri.host, @post_request_uri.port, proxy[:addr], proxy[:port], proxy[:user], proxy[:pass]) end + @http_client.use_ssl = true @log.info "POST Request url: #{@@post_request_url}" ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMPluginStart", {}) diff --git a/source/plugins/ruby/proxy_utils.rb b/source/plugins/ruby/proxy_utils.rb index 1566fe4e9..14fc47692 100644 --- a/source/plugins/ruby/proxy_utils.rb +++ b/source/plugins/ruby/proxy_utils.rb @@ -3,43 +3,63 @@ # frozen_string_literal: true class ProxyUtils - class << self - def getProxyConfiguration() - omsproxy_secret_path = "/etc/omsagent-secret/PROXY" - if !File.exist?(omsproxy_secret_path) - return {} - end - - begin - proxy_config = parseProxyConfiguration(File.read(omsproxy_secret_path)) - rescue SystemCallError # Error::ENOENT - return {} - end - - if proxy_config.nil? - $log.warn("Failed to parse the proxy configuration in '#{omsproxy_secret_path}'") - return {} - end - - return proxy_config - end - - def parseProxyConfiguration(proxy_conf_str) - # Remove the http(s) protocol - proxy_conf_str = proxy_conf_str.gsub(/^(https?:\/\/)?/, "") - - # Check for unsupported protocol - if proxy_conf_str[/^[a-z]+:\/\//] - return nil - end - - re = /^(?:(?[^:]+):(?[^@]+)@)?(?[^:@]+)(?::(?\d+))?$/ - matches = re.match(proxy_conf_str) - if matches.nil? or matches[:addr].nil? - return nil - end - # Convert nammed matches to a hash - Hash[ matches.names.map{ |name| name.to_sym}.zip( matches.captures ) ] + class << self + def getProxyConfiguration() + omsproxy_secret_path = "/etc/omsagent-secret/PROXY" + if !File.exist?(omsproxy_secret_path) + return {} + end + + begin + proxy_config = parseProxyConfiguration(File.read(omsproxy_secret_path)) + rescue SystemCallError # Error::ENOENT + return {} + end + + if proxy_config.nil? + $log.warn("Failed to parse the proxy configuration in '#{omsproxy_secret_path}'") + return {} + end + + return proxy_config + end + + def parseProxyConfiguration(proxy_conf_str) + if proxy_conf_str.empty? + return nil + end + # Remove trailing / if the proxy endpoint has + if proxy_conf_str.end_with?("/") + proxy_conf_str = proxy_conf_str.chop + end + # Remove the http(s) protocol + proxy_conf_str = proxy_conf_str.gsub(/^(https?:\/\/)?/, "") + + # Check for unsupported protocol + if proxy_conf_str[/^[a-z]+:\/\//] + return nil + end + + re = /^(?:(?[^:]+):(?[^@]+)@)?(?[^:@]+)(?::(?\d+))?$/ + matches = re.match(proxy_conf_str) + if matches.nil? or matches[:addr].nil? + return nil + end + # Convert nammed matches to a hash + Hash[matches.names.map { |name| name.to_sym }.zip(matches.captures)] + end + + def isProxyCACertConfigured() + isProxyCACertExist = false + begin + proxy_cert_path = "/etc/omsagent-secret/PROXYCERT.crt" + if File.exist?(proxy_cert_path) + isProxyCACertExist = true end + rescue => error + $log.warn("Failed to check the existence of Proxy CA cert '#{proxy_cert_path}'") + end + return isProxyCACertExist end -end \ No newline at end of file + end +end diff --git a/test/e2e/src/core/conftest.py b/test/e2e/src/core/conftest.py index 9fe34952c..02f644a18 100644 --- a/test/e2e/src/core/conftest.py +++ b/test/e2e/src/core/conftest.py @@ -87,6 +87,7 @@ def env_dict(): return print('Starting cleanup...') append_result_output("Starting Cleanup...\n", env_dict['SETUP_LOG_FILE']) + print("Cleanup Complete.") append_result_output("Cleanup Complete.\n", env_dict['SETUP_LOG_FILE']) return diff --git a/test/e2e/src/tests/test_node_metrics_e2e_workflow.py b/test/e2e/src/tests/test_node_metrics_e2e_workflow.py index 667c718b4..dfcc89dde 100755 --- a/test/e2e/src/tests/test_node_metrics_e2e_workflow.py +++ b/test/e2e/src/tests/test_node_metrics_e2e_workflow.py @@ -134,6 +134,7 @@ def test_node_metrics_e2e_workflow(env_dict): constants.NODE_MEMORY_RSS_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) if len(timeseries) <= 0: pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.NODE_MEMORY_RSS_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) + # node metric - memoryRssPercentage custommetricsUrl = '{0}{1}/providers/microsoft.Insights/metrics?timespan={2}/{3}&interval=FULL&metricnames={4}&aggregation={5}&metricNamespace={6}&validatedimensions=false&api-version={7}'.format( resourceManager.rstrip("/"), @@ -182,6 +183,7 @@ def test_node_metrics_e2e_workflow(env_dict): constants.NODE_MEMORY_RSS_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) if len(timeseries) <= 0: pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.NODE_MEMORY_RSS_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) + # node metric - memoryWorkingSetBytes custommetricsUrl = '{0}{1}/providers/microsoft.Insights/metrics?timespan={2}/{3}&interval=FULL&metricnames={4}&aggregation={5}&metricNamespace={6}&validatedimensions=false&api-version={7}'.format( resourceManager.rstrip("/"), @@ -229,6 +231,7 @@ def test_node_metrics_e2e_workflow(env_dict): constants.NODE_MEMORY_WS_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) if len(timeseries) <= 0: pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.NODE_MEMORYE_WS_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) + # node metric - memoryWorkingSetPercentage custommetricsUrl = '{0}{1}/providers/microsoft.Insights/metrics?timespan={2}/{3}&interval=FULL&metricnames={4}&aggregation={5}&metricNamespace={6}&validatedimensions=false&api-version={7}'.format( resourceManager.rstrip("/"), @@ -276,6 +279,7 @@ def test_node_metrics_e2e_workflow(env_dict): constants.NODE_MEMORY_WS_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) if len(timeseries) <= 0: pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.NODE_MEMORY_WS_PERCENTAGE_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) + # node metric - cpuUsageMilliCores custommetricsUrl = '{0}{1}/providers/microsoft.Insights/metrics?timespan={2}/{3}&interval=FULL&metricnames={4}&aggregation={5}&metricNamespace={6}&validatedimensions=false&api-version={7}'.format( resourceManager.rstrip("/"), @@ -322,6 +326,7 @@ def test_node_metrics_e2e_workflow(env_dict): constants.NODE_CPU_USAGE_MILLI_CORES_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) if len(timeseries) <= 0: pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.NODE_CPU_USAGE_MILLI_CORES_METRIC_NAME, constants.NODE_METRICS_NAMESPACE)) + # node metric - cpuUsagePercentage custommetricsUrl = '{0}{1}/providers/microsoft.Insights/metrics?timespan={2}/{3}&interval=FULL&metricnames={4}&aggregation={5}&metricNamespace={6}&validatedimensions=false&api-version={7}'.format( resourceManager.rstrip("/"), diff --git a/test/e2e/src/tests/test_pod_metrics_e2e_workflow.py b/test/e2e/src/tests/test_pod_metrics_e2e_workflow.py index 1eaca71ba..81e2b77a9 100755 --- a/test/e2e/src/tests/test_pod_metrics_e2e_workflow.py +++ b/test/e2e/src/tests/test_pod_metrics_e2e_workflow.py @@ -133,6 +133,7 @@ def test_pod_metrics_e2e_workflow(env_dict): constants.POD_COUNT_METRIC_NAME, constants.POD_METRICS_NAMESPACE)) if len(timeseries) <= 0: pytest.fail("length of timeseries should be greater than for 0 for metric: {0} in namespace :{1}".format(constants.POD_COUNT_METRIC_NAME, constants.POD_METRICS_NAMESPACE)) + append_result_output("test_pod_metrics_e2e_workflow end \n", env_dict['TEST_AGENT_LOG_FILE']) print("Successfully completed e2e workflows test.") diff --git a/test/e2e/src/tests/test_rs_workflows.py b/test/e2e/src/tests/test_rs_workflows.py index 1265f1b47..6a29dcc73 100755 --- a/test/e2e/src/tests/test_rs_workflows.py +++ b/test/e2e/src/tests/test_rs_workflows.py @@ -88,6 +88,7 @@ def test_rs_workflows(env_dict): if IsKubeContainerPerfInventorySuccessful == False: pytest.fail("KubeContainerPerfInventory stream not emitted successfully from pod:" + rspodName) + if IsKubeServicesInventorySuccessful == False: pytest.fail("KubeServicesInventory stream not emitted successfully from pod:" + rspodName)