microsoft · ganga1980 · Jan 13, 2021 · Aug 5, 2020 · Aug 6, 2020 · Aug 6, 2020
@@ -11,6 +11,8 @@ do
            KV) KV=$VALUE ;;
            KVSECRETNAMEAPPID) AppId=$VALUE ;;
            KVSECRETNAMEAPPSECRET) AppSecret=$VALUE ;;
+           KVSECRETNAMECDPXAPPID) CdpxAppId=$VALUE ;;
+           KVSECRETNAMECDPXAPPSECRET) CdpxAppSecret=$VALUE ;;
            *)
     esac
 done
@@ -27,4 +29,16 @@ az keyvault secret download --file ~/acrappsecret --vault-name ${KV}  --name ${A
 
 echo "downloaded the appsecret from KV:${KV} and KV secret:${AppSecret}"
 
+echo "key vault secret name for cdpx appid:${KVSECRETNAMECDPXAPPID}"
+
+echo "key vault secret name for cdpx appsecret:${KVSECRETNAMECDPXAPPSECRET}"
+
+az keyvault secret download --file ~/cdpxacrappid --vault-name ${KV}  --name ${CdpxAppId}
+
+echo "downloaded the appid from KV:${KV} and KV secret:${CdpxAppId}"
+
+az keyvault secret download --file ~/cdpxacrappsecret --vault-name ${KV}  --name ${CdpxAppSecret}
+
+echo "downloaded the appsecret from KV:${KV} and KV secret:${CdpxAppSecret}"
+
 echo "end: get app id and secret from specified key vault"
@@ -25,12 +25,21 @@ ACR_APP_ID=$(cat ~/acrappid)
 ACR_APP_SECRET=$(cat ~/acrappsecret)
 echo "end: read appid and appsecret"
 
+echo "start: read appid and appsecret for cdpx"
+CDPX_ACR_APP_ID=$(cat ~/cdpxacrappid)
+CDPX_ACR_APP_SECRET=$(cat ~/cdpxacrappsecret)
+echo "end: read appid and appsecret which has read access on cdpx acr"
+
+
+# Name of CDPX_ACR should be in this format :Naming convention: 'cdpx' + service tree id without '-' + two digit suffix like'00'/'01
+# suffix 00 primary and 01 secondary, and we only use primary
+# This configured via pipeline variable
 echo "login to cdpxlinux acr:${CDPX_ACR}"
-docker login $CDPX_ACR  --username $ACR_APP_ID --password $ACR_APP_SECRET
+docker login $CDPX_ACR  --username $CDPX_ACR_APP_ID --password $CDPX_ACR_APP_SECRET
 echo "login to cdpxlinux acr completed: ${CDPX_ACR}"
 
 echo "pull agent image from cdpxlinux acr: ${CDPX_ACR}"
-docker pull ${CDPX_ACR}/artifact/3170cdd2-19f0-4027-912b-1027311691a2/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG}
+docker pull ${CDPX_ACR}/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG}
 echo "pull image from cdpxlinux acr completed: ${CDPX_ACR}"
 
 echo "CI Release name is:"$CI_RELEASE
@@ -41,7 +50,7 @@ echo "CI ACR : ${CI_ACR}"
 echo "CI AGENT REPOSITORY NAME : ${CI_AGENT_REPO}"
 
 echo "tag linux agent image"
-docker tag ${CDPX_ACR}/artifact/3170cdd2-19f0-4027-912b-1027311691a2/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG} ${CI_ACR}/public/azuremonitor/containerinsights/${CI_AGENT_REPO}:${imagetag}
+docker tag ${CDPX_ACR}/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG} ${CI_ACR}/public/azuremonitor/containerinsights/${CI_AGENT_REPO}:${imagetag}
 
 echo "login ciprod acr":$CI_ACR
 docker login $CI_ACR --username $ACR_APP_ID --password $ACR_APP_SECRET

@@ -25,12 +25,20 @@ ACR_APP_ID=$(cat ~/acrappid )
 ACR_APP_SECRET=$(cat ~/acrappsecret)
 echo "end: read appid and appsecret"
 
+echo "start: read appid and appsecret for cdpx"
+CDPX_ACR_APP_ID=$(cat ~/cdpxacrappid)
+CDPX_ACR_APP_SECRET=$(cat ~/cdpxacrappsecret)
+echo "end: read appid and appsecret which has read access on cdpx acr"
+
+# Name of CDPX_ACR should be in this format :Naming convention: 'cdpx' + service tree id without '-' + two digit suffix like'00'/'01
+# suffix 00 primary and 01 secondary, and we only use primary
+# This configured via pipeline variable
 echo "login to cdpxwindows acr:${CDPX_ACR}"
-docker login $CDPX_ACR  --username $ACR_APP_ID --password $ACR_APP_SECRET
+docker login $CDPX_ACR  --username $CDPX_ACR_APP_ID --password $CDPX_ACR_APP_SECRET
 echo "login to cdpxwindows acr:${CDPX_ACR} completed"
 
 echo "pull image from cdpxwin acr: ${CDPX_ACR}"
-docker pull ${CDPX_ACR}/artifact/3170cdd2-19f0-4027-912b-1027311691a2/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG}
+docker pull ${CDPX_ACR}/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG}
 echo "pull image from cdpxwin acr completed: ${CDPX_ACR}"
 
 echo "CI Release name:"$CI_RELEASE
@@ -40,7 +48,7 @@ imagetag="win-"$CI_RELEASE$CI_IMAGE_TAG_SUFFIX
 echo "agentimagetag="$imagetag
 
 echo "tag windows agent image"
-docker tag ${CDPX_ACR}/artifact/3170cdd2-19f0-4027-912b-1027311691a2/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG} ${CI_ACR}/public/azuremonitor/containerinsights/${CI_AGENT_REPO}:${imagetag}
+docker tag ${CDPX_ACR}/official/${CDPX_REPO_NAME}:${CDPX_AGENT_IMAGE_TAG} ${CI_ACR}/public/azuremonitor/containerinsights/${CI_AGENT_REPO}:${imagetag}
 
 echo "login to ${CI_ACR} acr"
 docker login $CI_ACR --username $ACR_APP_ID --password $ACR_APP_SECRET

@@ -10,6 +10,84 @@ additional questions or comments.
 ## Release History
 
 Note : The agent version(s) below has dates (ciprod<mmddyyyy>), which indicate the agent build dates (not release dates)
+### 01/11/2021 -
+##### Version microsoft/oms:ciprod01112021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod01112021 (linux)
+##### Version microsoft/oms:win-ciprod01112021 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod01112021 (windows)
+##### Code change log
+- Fixes for Linux Agent Replicaset Pod OOMing issue
+- Update fluentbit (1.14.2 to 1.6.8) for the Linux Daemonset 
+- Make Fluentbit settings: log_flush_interval_secs, tail_buf_chunksize_megabytes and tail_buf_maxsize_megabytes configurable via configmap
+- Support for PV inventory collection
+- Removal of Custom metric region check for Public cloud regions and update to use cloud environment variable to determine the custom metric support
+- For daemonset pods, add the dnsconfig to use ndots: 3 from ndots:5 to optimize the number of DNS API calls made
+- Fix for inconsistency in the collection container environment variables for the pods which has high number of containers
+- Fix for disabling of std{out;err} log_collection_settings via configmap issue in windows daemonset 
+- Update to use workspace key from mount file rather than environment variable for windows daemonset agent
+- Remove per container info logs in the container inventory
+- Enable ADX route for windows container logs
+- Remove logging to termination log in windows agent liveness probe
+
+
+### 11/09/2020 -
+##### Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod11092020 (linux)
+##### Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod11092020 (windows)
+##### Code change log
+- Fix for duplicate windows metrics
+
+### 10/27/2020 -
+##### Version microsoft/oms:ciprod10272020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod10272020 (linux)
+##### Version microsoft/oms:win-ciprod10272020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod10052020 (windows)
+##### Code change log
+- Activate oneagent in few AKS regions (koreacentral,norwayeast)
+- Disable syslog
+- Fix timeout for Windows daemonset liveness probe
+- Make request == limit for Windows daemonset resources (cpu & memory)
+- Schema v2 for container log (ADX only - applicable only for select customers for piloting)
+
+### 10/05/2020 -
+##### Version microsoft/oms:ciprod10052020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod10052020 (linux)
+##### Version microsoft/oms:win-ciprod10052020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod10052020 (windows)
+##### Code change log
+- Health CRD to version v1 (from v1beta1) for k8s versions >= 1.19.0
+- Collection of PV usage metrics for PVs mounted by pods (kube-system pods excluded by default)(doc-link-needed)
+- Zero fill few custom metrics under a timer, also add zero filling for new PV usage metrics
+- Collection of additional Kubelet metrics ('kubelet_running_pod_count','volume_manager_total_volumes','kubelet_node_config_error','process_resident_memory_bytes','process_cpu_seconds_total','kubelet_runtime_operations_total','kubelet_runtime_operations_errors_total'). This also includes updates to 'kubelet' workbook to include these new metrics
+- Collection of Azure NPM (Network Policy Manager) metrics (basic & advanced. By default, NPM metrics collection is turned OFF)(doc-link-needed)
+- Support log collection when docker root is changed with knode. Tracked by [this](https://github.com/Azure/AKS/issues/1373) issue
+- Support for Pods in 'Terminating' state for nodelost scenarios
+- Fix for reduction in telemetry for custom metrics ingestion failures
+- Fix CPU capacity/limits metrics being 0 for Virtual nodes (VK)
+- Add new custom metric regions (eastus2,westus,australiasoutheast,brazilsouth,germanywestcentral,northcentralus,switzerlandnorth)
+- Enable strict SSL validation for AppInsights Ruby SDK
+- Turn off custom metrics upload for unsupported cluster types
+- Install CA certs from wire server for windows (in certain clouds)
+
+### 09/16/2020 -
+> Note: This agent release targetted ONLY for non-AKS clusters via Azure Monitor for containers HELM chart update
+##### Version microsoft/oms:ciprod09162020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod09162020 (linux)
+##### Version microsoft/oms:win-ciprod09162020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod09162020 (windows)
+##### Code change log
+- Collection of Azure Network Policy Manager Basic and Advanced metrics
+- Add support in Windows Agent for Container log collection of CRI runtimes such as ContainerD
+- Alertable metrics support Arc K8s cluster to parity with AKS
+- Support for multiple container log mount paths when docker is updated through knode
+- Bug fix related to MDM telemetry
+
+### 08/07/2020 -
+##### Version microsoft/oms:ciprod08072020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod08072020 (linux)
+##### Version microsoft/oms:win-ciprod08072020 Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:win-ciprod08072020 (windows)
+##### Code change log
+- Collection of KubeState metrics for deployments and HPA
+- Add the Proxy support for Windows agent
+- Fix for ContainerState in ContainerInventory to handle Failed state and collection of environment variables for terminated and failed containers
+- Change /spec to /metrics/cadvisor endpoint to collect node capacity metrics
+- Disable Health Plugin by default and can enabled via configmap
+- Pin version of jq to 1.5+dfsg-2
+- Bug fix for showing node as 'not ready' when there is disk pressure
+- oneagent integration (disabled by default)
+- Add region check before sending alertable metrics to MDM
+- Telemetry fix for agent telemetry for sov. clouds
+
 
 ### 11/09/2020 -
 ##### Version mcr.microsoft.com/azuremonitor/containerinsights/ciprod:ciprod11092020 (linux)

@@ -18,12 +18,17 @@ def substituteFluentBitPlaceHolders
     bufferChunkSize = ENV["FBIT_TAIL_BUFFER_CHUNK_SIZE"]
     bufferMaxSize = ENV["FBIT_TAIL_BUFFER_MAX_SIZE"]
 
-    serviceInterval = (!interval.nil? && is_number?(interval)) ? interval : @default_service_interval
+    serviceInterval = (!interval.nil? && is_number?(interval) && interval.to_i > 0 ) ? interval : @default_service_interval
     serviceIntervalSetting = "Flush         " + serviceInterval
 
-    tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize)) ? bufferChunkSize : nil
+    tailBufferChunkSize = (!bufferChunkSize.nil? && is_number?(bufferChunkSize) && bufferChunkSize.to_i > 0) ? bufferChunkSize : nil
 
-    tailBufferMaxSize = (!bufferMaxSize.nil? && is_number?(bufferMaxSize)) ? bufferMaxSize : nil
+    tailBufferMaxSize = (!bufferMaxSize.nil? && is_number?(bufferMaxSize) && bufferMaxSize.to_i > 0) ? bufferMaxSize : nil
+
+    if ((!tailBufferChunkSize.nil? && tailBufferMaxSize.nil?) ||  (!tailBufferChunkSize.nil? && !tailBufferMaxSize.nil? && tailBufferChunkSize.to_i > tailBufferMaxSize.to_i))
+      puts "config:warn buffer max size must be greater or equal to chunk size"
+      tailBufferMaxSize = tailBufferChunkSize
+    end
 
     text = File.read(@td_agent_bit_conf_path)
     new_contents = text.gsub("${SERVICE_FLUSH_INTERVAL}", serviceIntervalSetting)

@@ -228,7 +228,7 @@ def get_command_windows(env_variable_name, env_variable_value)
     file.write(commands)
     commands = get_command_windows('AZMON_LOG_TAIL_PATH', @logTailPath)
     file.write(commands)
-    commands = get_command_windows('AZMON_LOG_EXCLUSION_REGEX_PATTERN', @stdoutExcludeNamespaces)
+    commands = get_command_windows('AZMON_LOG_EXCLUSION_REGEX_PATTERN', @logExclusionRegexPattern)
     file.write(commands)
     commands = get_command_windows('AZMON_STDOUT_EXCLUDED_NAMESPACES', @stdoutExcludeNamespaces)
     file.write(commands)
@@ -244,7 +244,7 @@ def get_command_windows(env_variable_name, env_variable_value)
     file.write(commands)
     commands = get_command_windows('AZMON_CLUSTER_COLLECT_ALL_KUBE_EVENTS', @collectAllKubeEvents)
     file.write(commands)
-    commands = get_command_windows('AZMON_CONTAINER_LOGS_ROUTE', @containerLogsRoute)
+    commands = get_command_windows('AZMON_CONTAINER_LOGS_EFFECTIVE_ROUTE', @containerLogsRoute)
     file.write(commands)
 
     # Close file after writing all environment variables

@@ -45,14 +45,12 @@
 #custom_metrics_mdm filter plugin
 <filter mdm.cadvisorperf**>
   type filter_cadvisor2mdm
-  custom_metrics_azure_regions eastus,southcentralus,westcentralus,westus2,southeastasia,northeurope,westeurope,southafricanorth,centralus,northcentralus,eastus2,koreacentral,eastasia,centralindia,uksouth,canadacentral,francecentral,japaneast,australiaeast,eastus2,westus,australiasoutheast,brazilsouth,germanywestcentral,northcentralus,switzerlandnorth
   metrics_to_collect cpuUsageNanoCores,memoryWorkingSetBytes,memoryRssBytes,pvUsedBytes
   log_level info
 </filter>
 
 <filter oms.mdm.container.perf.telegraf**>
   type filter_telegraf2mdm
-  custom_metrics_azure_regions eastus,southcentralus,westcentralus,westus2,southeastasia,northeurope,westeurope,southafricanorth,centralus,northcentralus,eastus2,koreacentral,eastasia,centralindia,uksouth,canadacentral,francecentral,japaneast,australiaeast,eastus2,westus,australiasoutheast,brazilsouth,germanywestcentral,northcentralus,switzerlandnorth
   log_level debug
 </filter>
 

@@ -13,7 +13,14 @@
      tag oms.containerinsights.KubePodInventory
      run_interval 60
      log_level debug
-     custom_metrics_azure_regions eastus,southcentralus,westcentralus,westus2,southeastasia,northeurope,westeurope,southafricanorth,centralus,northcentralus,eastus2,koreacentral,eastasia,centralindia,uksouth,canadacentral,francecentral,japaneast,australiaeast,eastus2,westus,australiasoutheast,brazilsouth,germanywestcentral,northcentralus,switzerlandnorth
+    </source>
+
+    #Kubernetes Persistent Volume inventory
+    <source>
+     type kubepvinventory
+     tag oms.containerinsights.KubePVInventory
+     run_interval 60
+     log_level debug
     </source>
 
     #Kubernetes events
@@ -66,14 +73,12 @@
 
     <filter mdm.kubenodeinventory**>
      type filter_inventory2mdm
-     custom_metrics_azure_regions eastus,southcentralus,westcentralus,westus2,southeastasia,northeurope,westeurope,southafricanorth,centralus,northcentralus,eastus2,koreacentral,eastasia,centralindia,uksouth,canadacentral,francecentral,japaneast,australiaeast,eastus2,westus,australiasoutheast,brazilsouth,germanywestcentral,northcentralus,switzerlandnorth
      log_level info
     </filter>
 
     #custom_metrics_mdm filter plugin for perf data from windows nodes
     <filter mdm.cadvisorperf**>
      type filter_cadvisor2mdm
-     custom_metrics_azure_regions eastus,southcentralus,westcentralus,westus2,southeastasia,northeurope,westeurope,southafricanorth,centralus,northcentralus,eastus2,koreacentral,eastasia,centralindia,uksouth,canadacentral,francecentral,japaneast,australiaeast,eastus2,westus,australiasoutheast,brazilsouth,germanywestcentral,northcentralus,switzerlandnorth
      metrics_to_collect cpuUsageNanoCores,memoryWorkingSetBytes,pvUsedBytes
      log_level info
     </filter>
@@ -98,6 +103,21 @@
      max_retry_wait 5m
     </match>
 
+    <match oms.containerinsights.KubePVInventory**>
+     type out_oms
+     log_level debug
+     num_threads 5
+     buffer_chunk_limit 4m
+     buffer_type file
+     buffer_path %STATE_DIR_WS%/state/out_oms_kubepv*.buffer
+     buffer_queue_limit 20
+     buffer_queue_full_action drop_oldest_chunk
+     flush_interval 20s
+     retry_limit 10
+     retry_wait 5s
+     max_retry_wait 5m
+    </match>
+
     <match oms.containerinsights.KubeEvents**>
      type out_oms
      log_level debug

@@ -22,6 +22,7 @@ MAINTAINER:              'Microsoft Corporation'
 /opt/microsoft/omsagent/plugin/filter_container.rb;			    source/plugins/ruby/filter_container.rb;				644; root; root
 
 /opt/microsoft/omsagent/plugin/in_kube_podinventory.rb;			source/plugins/ruby/in_kube_podinventory.rb;			644; root; root
+/opt/microsoft/omsagent/plugin/in_kube_pvinventory.rb;			source/plugins/ruby/in_kube_pvinventory.rb;			644; root; root
 /opt/microsoft/omsagent/plugin/in_kube_events.rb;			    source/plugins/ruby/in_kube_events.rb;				644; root; root
 /opt/microsoft/omsagent/plugin/KubernetesApiClient.rb;			source/plugins/ruby/KubernetesApiClient.rb;			644; root; root
 
@@ -122,7 +123,7 @@ MAINTAINER:              'Microsoft Corporation'
 /opt/tomlparser-mdm-metrics-config.rb;                          build/linux/installer/scripts/tomlparser-mdm-metrics-config.rb;     755; root; root
 /opt/tomlparser-metric-collection-config.rb;                    build/linux/installer/scripts/tomlparser-metric-collection-config.rb;     755; root; root
 
-/opt/tomlparser-health-config.rb;                               build/linux/installer/scripts/tomlparser-health-config.rb;     755; root; root
+/opt/tomlparser-agent-config.rb;                                build/linux/installer/scripts/tomlparser-agent-config.rb;     755; root; root
 /opt/tomlparser.rb;                                             build/common/installer/scripts/tomlparser.rb;     755; root; root
 /opt/td-agent-bit-conf-customizer.rb;                           build/common/installer/scripts/td-agent-bit-conf-customizer.rb;     755; root; root
 /opt/ConfigParseErrorLogger.rb;                                 build/common/installer/scripts/ConfigParseErrorLogger.rb;           755; root; root