From 5b662f702edb822648f09394aea2093fde5cc9cb Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 00:03:51 -0700 Subject: [PATCH 01/20] wip --- kubernetes/omsagent.yaml | 4 ++ kubernetes/windows/main.ps1 | 103 ++++++++++++++++++++++-------------- 2 files changed, 67 insertions(+), 40 deletions(-) diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index 29533e678..db788a37e 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -660,6 +660,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP volumeMounts: - mountPath: C:\ProgramData\docker\containers name: docker-windows-containers diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index b7ddfa8e7..799eb51d1 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -1,34 +1,28 @@ -function Confirm-WindowsServiceExists($name) -{ - if (Get-Service $name -ErrorAction SilentlyContinue) - { +function Confirm-WindowsServiceExists($name) { + if (Get-Service $name -ErrorAction SilentlyContinue) { return $true } return $false } -function Remove-WindowsServiceIfItExists($name) -{ +function Remove-WindowsServiceIfItExists($name) { $exists = Confirm-WindowsServiceExists $name - if ($exists) - { + if ($exists) { sc.exe \\server delete $name } } -function Start-FileSystemWatcher -{ +function Start-FileSystemWatcher { Start-Process powershell -NoNewWindow .\filesystemwatcher.ps1 } #register fluentd as a windows service -function Set-EnvironmentVariables -{ +function Set-EnvironmentVariables { $domain = "opinsights.azure.com" if (Test-Path /etc/omsagent-secret/DOMAIN) { # TODO: Change to omsagent-secret before merging - $domain = Get-Content /etc/omsagent-secret/DOMAIN + $domain = Get-Content /etc/omsagent-secret/DOMAIN } # Set DOMAIN @@ -38,7 +32,7 @@ function Set-EnvironmentVariables $wsID = "" if (Test-Path /etc/omsagent-secret/WSID) { # TODO: Change to omsagent-secret before merging - $wsID = Get-Content /etc/omsagent-secret/WSID + $wsID = Get-Content /etc/omsagent-secret/WSID } # Set DOMAIN @@ -48,7 +42,7 @@ function Set-EnvironmentVariables $wsKey = "" if (Test-Path /etc/omsagent-secret/KEY) { # TODO: Change to omsagent-secret before merging - $wsKey = Get-Content /etc/omsagent-secret/KEY + $wsKey = Get-Content /etc/omsagent-secret/KEY } # Set KEY @@ -58,7 +52,7 @@ function Set-EnvironmentVariables $proxy = "" if (Test-Path /etc/omsagent-secret/PROXY) { # TODO: Change to omsagent-secret before merging - $proxy = Get-Content /etc/omsagent-secret/PROXY + $proxy = Get-Content /etc/omsagent-secret/PROXY Write-Host "Validating the proxy configuration since proxy configuration provided" # valide the proxy endpoint configuration if (![string]::IsNullOrEmpty($proxy)) { @@ -66,26 +60,22 @@ function Set-EnvironmentVariables if (![string]::IsNullOrEmpty($proxy)) { $proxy = [string]$proxy.Trim(); $parts = $proxy -split "@" - if ($parts.Length -ne 2) - { + if ($parts.Length -ne 2) { Write-Host "Invalid ProxyConfiguration $($proxy). EXITING....." exit 1 } $subparts1 = $parts[0] -split "//" - if ($subparts1.Length -ne 2) - { + if ($subparts1.Length -ne 2) { Write-Host "Invalid ProxyConfiguration $($proxy). EXITING....." exit 1 } $protocol = $subparts1[0].ToLower().TrimEnd(":") - if (!($protocol -eq "http") -and !($protocol -eq "https")) - { + if (!($protocol -eq "http") -and !($protocol -eq "https")) { Write-Host "Unsupported protocol in ProxyConfiguration $($proxy). EXITING....." exit 1 } $subparts2 = $parts[1] -split ":" - if ($subparts2.Length -ne 2) - { + if ($subparts2.Length -ne 2) { Write-Host "Invalid ProxyConfiguration $($proxy). EXITING....." exit 1 } @@ -118,12 +108,51 @@ function Set-EnvironmentVariables .\setenv.ps1 } -function Start-Fluent -{ +function Start-Fluent { + $containerRuntime = "docker" # Run fluent-bit service first so that we do not miss any logs being forwarded by the fluentd service. # Run fluent-bit as a background job. Switch this to a windows service once fluent-bit supports natively running as a windows service Start-Job -ScriptBlock { Start-Process -NoNewWindow -FilePath "C:\opt\fluent-bit\bin\fluent-bit.exe" -ArgumentList @("-c", "C:\etc\fluent-bit\fluent-bit.conf", "-e", "C:\opt\omsagentwindows\out_oms.so") } + # determine the container runtime + $response = Invoke-WebRequest -uri http://$NODE_IP:10255/pods -UseBasicParsing + $isPodsAPISuccess = $flase + + if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { + Write-Host "Response of the Invoke-WebRequest -uri http://$NODE_IP:10255/pods is : $($response.StatusCode)" + $isPodsAPISuccess = $true + } + else { + $response = Invoke-WebRequest -Uri https://$NODE_IP:10250/pods -Headers @{'Authorization' = "Bearer $(Get-Content /var/run/secrets/kubernetes.io/serviceaccount/token)" } -UseBasicParsing + if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { + Write-Host "Response of the Invoke-WebRequest -uri http://$NODE_IP:10250/pods is : $($response.StatusCode)" + $isPodsAPISuccess = $true + } + } + if ($isPodsAPISuccess -and ![string]::IsNullOrEmpty($response.Content)) { + $podList = $response.Content | ConvertFrom-Json + if (![string]::IsNullOrEmpty($podList)) { + $podItems = $podList.Items + if (![string]::IsNullOrEmpty($podItems) -and $podItems.Length -gt 0) { + Write-Host "found pod items: $($podItems.Length)" + for ($index = 0; $index -le $podItems.Length ; $index++) { + Write-Host "podItem index : $($index)" + $pod = $podItems[$index] + if (![string]::IsNullOrEmpty($pod) -and + ![string]::IsNullOrEmpty($pod.status) -and + ![string]::IsNullOrEmpty($pod.status.phase) -and + $pod.status.phase -eq "Running" -and + $pod.status.ContainerStatuses.Length -gt 0) { + $containerID = $pod.status.ContainerStatuses[0].containerID + $containerRuntime = $containerID.split(":")[0].trim() + Write-Host "detected containerRuntime as : $($containerRuntime)" + break + } + } + } + } + } + #register fluentd as a service and start # there is a known issues with win32-service https://github.com/chef/win32-service/issues/70 fluentd --reg-winsvc i --reg-winsvc-auto-start --winsvc-name fluentdwinaks --reg-winsvc-fluentdopt '-c C:/etc/fluent/fluent.conf -o C:/etc/fluent/fluent.log' @@ -131,33 +160,27 @@ function Start-Fluent Notepad.exe | Out-Null } -function Generate-Certificates -{ +function Generate-Certificates { Write-Host "Generating Certificates" C:\\opt\\omsagentwindows\\certgenerator\\certificategenerator.exe } -function Test-CertificatePath -{ +function Test-CertificatePath { $certLocation = $env:CI_CERT_LOCATION - $keyLocation = $env:CI_KEY_LOCATION - if (!(Test-Path $certLocation)) - { + $keyLocation = $env:CI_KEY_LOCATION + if (!(Test-Path $certLocation)) { Write-Host "Certificate file not found at $($certLocation). EXITING....." exit 1 } - else - { + else { Write-Host "Certificate file found at $($certLocation)" } - if (! (Test-Path $keyLocation)) - { + if (! (Test-Path $keyLocation)) { Write-Host "Key file not found at $($keyLocation). EXITING...." exit 1 } - else - { + else { Write-Host "Key file found at $($keyLocation)" } } @@ -172,7 +195,7 @@ Test-CertificatePath Start-Fluent # List all powershell processes running. This should have main.ps1 and filesystemwatcher.ps1 -Get-WmiObject Win32_process | Where-Object {$_.Name -match 'powershell'} | Format-Table -Property Name, CommandLine, ProcessId +Get-WmiObject Win32_process | Where-Object { $_.Name -match 'powershell' } | Format-Table -Property Name, CommandLine, ProcessId #check if fluentd service is running Get-Service fluentdwinaks From 28c0d0e631ccf3ec4ba3b3dbe2e22e2dbb620292 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 13:20:56 -0700 Subject: [PATCH 02/20] wip --- build/windows/installer/conf/fluent-cri.conf | 73 ++++++++++++ kubernetes/windows/Dockerfile | 2 + kubernetes/windows/main.ps1 | 111 +++++++++++++------ 3 files changed, 152 insertions(+), 34 deletions(-) create mode 100644 build/windows/installer/conf/fluent-cri.conf diff --git a/build/windows/installer/conf/fluent-cri.conf b/build/windows/installer/conf/fluent-cri.conf new file mode 100644 index 000000000..5919110dc --- /dev/null +++ b/build/windows/installer/conf/fluent-cri.conf @@ -0,0 +1,73 @@ + + type heartbeat_request + run_interval 30m + @log_level info + + + + @type tail + path /var/log/containers/*.log + pos_file /var/opt/microsoft/fluent/fluentd-containers.log.pos + tag oms.container.log.la + @log_level trace + path_key tailed_path + limit_recently_modified 5m + + @type regexp + expression ^(? + + + + @type tail + path /var/log/containers/omsagent*.log + pos_file /opt/microsoft/fluent/omsagent-fluentd-containers.log.pos + tag oms.container.log.flbplugin + @log_level trace + path_key tailed_path + read_from_head true + + @type regexp + expression ^(? + + + + @type record_transformer + # fluent-plugin-record-modifier more light-weight but needs to be installed (dependency worth it?) + remove_keys tailed_path + + filepath ${record["tailed_path"]} + + + + + + @type forward + send_timeout 60s + recover_wait 10s + hard_timeout 60s + heartbeat_type none + ignore_network_errors_at_startup true + + name logaggregationserver + host 127.0.0.1 + port 25230 + weight 60 + + + + overflow_action throw_exception + chunk_limit_size 32k + queued_chunks_limit_size 256 + flush_interval 1 + flush_thread_interval 0.5 + flush_thread_burst_interval 0.01 + flush_thread_count 4 + retry_forever true + + diff --git a/kubernetes/windows/Dockerfile b/kubernetes/windows/Dockerfile index c8162b539..7979e8a3a 100644 --- a/kubernetes/windows/Dockerfile +++ b/kubernetes/windows/Dockerfile @@ -56,6 +56,8 @@ COPY ./omsagentwindows/out_oms.so /opt/omsagentwindows/out_oms.so # copy fluent, fluent-bit and out_oms conf files COPY ./omsagentwindows/installer/conf/fluent.conf /etc/fluent/ +# copy fluent cri runtime conf file +COPY ./omsagentwindows/installer/conf/fluent-cri.conf /etc/fluent/ COPY ./omsagentwindows/installer/conf/fluent-bit.conf /etc/fluent-bit COPY ./omsagentwindows/installer/conf/out_oms.conf /etc/omsagentwindows diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 799eb51d1..7d772f08a 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -1,3 +1,14 @@ +add-type @" +using System.Net; +using System.Security.Cryptography.X509Certificates; +public class TrustAllCertsPolicy : ICertificatePolicy { + public bool CheckValidationResult( + ServicePoint srvPoint, X509Certificate certificate, + WebRequest request, int certificateProblem) { + return true; + } +} +"@ function Confirm-WindowsServiceExists($name) { if (Get-Service $name -ErrorAction SilentlyContinue) { return $true @@ -108,54 +119,86 @@ function Set-EnvironmentVariables { .\setenv.ps1 } -function Start-Fluent { +function Get-ContainerRuntime { $containerRuntime = "docker" - # Run fluent-bit service first so that we do not miss any logs being forwarded by the fluentd service. - # Run fluent-bit as a background job. Switch this to a windows service once fluent-bit supports natively running as a windows service - Start-Job -ScriptBlock { Start-Process -NoNewWindow -FilePath "C:\opt\fluent-bit\bin\fluent-bit.exe" -ArgumentList @("-c", "C:\etc\fluent-bit\fluent-bit.conf", "-e", "C:\opt\omsagentwindows\out_oms.so") } - # determine the container runtime - $response = Invoke-WebRequest -uri http://$NODE_IP:10255/pods -UseBasicParsing - $isPodsAPISuccess = $flase - - if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { - Write-Host "Response of the Invoke-WebRequest -uri http://$NODE_IP:10255/pods is : $($response.StatusCode)" - $isPodsAPISuccess = $true + $NODE_IP = "" + if (![string]::IsNullOrEmpty([System.Environment]::GetEnvironmentVariable("NODE_IP", "PROCESS"))) { + $NODE_IP = [System.Environment]::GetEnvironmentVariable("NODE_IP", "PROCESS") } - else { - $response = Invoke-WebRequest -Uri https://$NODE_IP:10250/pods -Headers @{'Authorization' = "Bearer $(Get-Content /var/run/secrets/kubernetes.io/serviceaccount/token)" } -UseBasicParsing + elseif (![string]::IsNullOrEmpty([System.Environment]::GetEnvironmentVariable("NODE_IP", "USER"))) { + $NODE_IP = [System.Environment]::GetEnvironmentVariable("NODE_IP", "USER") + } + elseif (![string]::IsNullOrEmpty([System.Environment]::GetEnvironmentVariable("NODE_IP", "MACHINE"))) { + $NODE_IP = [System.Environment]::GetEnvironmentVariable("NODE_IP", "MACHINE") + } + + if (![string]::IsNullOrEmpty($NODE_IP)) { + Write-Host "Value of NODE_IP environment variable : $($NODE_IP)" + $response = Invoke-WebRequest -uri http://$NODE_IP:10255/pods -UseBasicParsing + $isPodsAPISuccess = $false + if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { - Write-Host "Response of the Invoke-WebRequest -uri http://$NODE_IP:10250/pods is : $($response.StatusCode)" + Write-Host "Response of the Invoke-WebRequest -uri http://$NODE_IP:10255/pods is : $($response.StatusCode)" $isPodsAPISuccess = $true } - } - if ($isPodsAPISuccess -and ![string]::IsNullOrEmpty($response.Content)) { - $podList = $response.Content | ConvertFrom-Json - if (![string]::IsNullOrEmpty($podList)) { - $podItems = $podList.Items - if (![string]::IsNullOrEmpty($podItems) -and $podItems.Length -gt 0) { - Write-Host "found pod items: $($podItems.Length)" - for ($index = 0; $index -le $podItems.Length ; $index++) { - Write-Host "podItem index : $($index)" - $pod = $podItems[$index] - if (![string]::IsNullOrEmpty($pod) -and - ![string]::IsNullOrEmpty($pod.status) -and - ![string]::IsNullOrEmpty($pod.status.phase) -and - $pod.status.phase -eq "Running" -and - $pod.status.ContainerStatuses.Length -gt 0) { - $containerID = $pod.status.ContainerStatuses[0].containerID - $containerRuntime = $containerID.split(":")[0].trim() - Write-Host "detected containerRuntime as : $($containerRuntime)" - break + else { + [System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Ssl3, [Net.SecurityProtocolType]::Tls, [Net.SecurityProtocolType]::Tls11, [Net.SecurityProtocolType]::Tls12 + $response = Invoke-WebRequest -Uri https://$NODE_IP:10250/pods -Headers @{'Authorization' = "Bearer $(Get-Content /var/run/secrets/kubernetes.io/serviceaccount/token)" } -UseBasicParsing + if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { + Write-Host "Response of the Invoke-WebRequest -uri https://$NODE_IP:10250/pods is : $($response.StatusCode)" + $isPodsAPISuccess = $true + } + } + + if ($isPodsAPISuccess -and ![string]::IsNullOrEmpty($response.Content)) { + $podList = $response.Content | ConvertFrom-Json + if (![string]::IsNullOrEmpty($podList)) { + $podItems = $podList.Items + if (![string]::IsNullOrEmpty($podItems) -and $podItems.Length -gt 0) { + Write-Host "found pod items: $($podItems.Length)" + for ($index = 0; $index -le $podItems.Length ; $index++) { + Write-Host "podItem index : $($index)" + $pod = $podItems[$index] + if (![string]::IsNullOrEmpty($pod) -and + ![string]::IsNullOrEmpty($pod.status) -and + ![string]::IsNullOrEmpty($pod.status.phase) -and + $pod.status.phase -eq "Running" -and + $pod.status.ContainerStatuses.Length -gt 0) { + $containerID = $pod.status.ContainerStatuses[0].containerID + $detectedContainerRuntime = $containerID.split(":")[0].trim() + if (![string]::IsNullOrEmpty($detectedContainerRuntime) -and [string]$detectedContainerRuntime.StartsWith('docker') -eq $false) { + $containerRuntime = $detectedContainerRuntime + Write-Host "detected containerRuntime as : $($containerRuntime)" + } + break + } } } } } } + return $containerRuntime +} + +function Start-Fluent { + + # Run fluent-bit service first so that we do not miss any logs being forwarded by the fluentd service. + # Run fluent-bit as a background job. Switch this to a windows service once fluent-bit supports natively running as a windows service + Start-Job -ScriptBlock { Start-Process -NoNewWindow -FilePath "C:\opt\fluent-bit\bin\fluent-bit.exe" -ArgumentList @("-c", "C:\etc\fluent-bit\fluent-bit.conf", "-e", "C:\opt\omsagentwindows\out_oms.so") } + + $containerRuntime = Get-ContainerRuntime + #register fluentd as a service and start # there is a known issues with win32-service https://github.com/chef/win32-service/issues/70 - fluentd --reg-winsvc i --reg-winsvc-auto-start --winsvc-name fluentdwinaks --reg-winsvc-fluentdopt '-c C:/etc/fluent/fluent.conf -o C:/etc/fluent/fluent.log' + if (![string]::IsNullOrEmpty($containerRuntime) -and [string]$containerRuntime.StartsWith('docker') -eq $false) { + fluentd --reg-winsvc i --reg-winsvc-auto-start --winsvc-name fluentdwinaks --reg-winsvc-fluentdopt '-c C:/etc/fluent/fluent-cri.conf -o C:/etc/fluent/fluent.log' + } + else { + fluentd --reg-winsvc i --reg-winsvc-auto-start --winsvc-name fluentdwinaks --reg-winsvc-fluentdopt '-c C:/etc/fluent/fluent.conf -o C:/etc/fluent/fluent.log' + } Notepad.exe | Out-Null } From 15bb1cd84fb04d2d722472ef60a15ca64c34d75c Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 13:25:43 -0700 Subject: [PATCH 03/20] wip --- kubernetes/windows/main.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 7d772f08a..0bda34d7b 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -121,7 +121,6 @@ function Set-EnvironmentVariables { function Get-ContainerRuntime { $containerRuntime = "docker" - # determine the container runtime $NODE_IP = "" if (![string]::IsNullOrEmpty([System.Environment]::GetEnvironmentVariable("NODE_IP", "PROCESS"))) { $NODE_IP = [System.Environment]::GetEnvironmentVariable("NODE_IP", "PROCESS") @@ -143,8 +142,9 @@ function Get-ContainerRuntime { $isPodsAPISuccess = $true } else { - [System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy - [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Ssl3, [Net.SecurityProtocolType]::Tls, [Net.SecurityProtocolType]::Tls11, [Net.SecurityProtocolType]::Tls12 + # set the certificate policy to ignore the certificate policy since kubelet uses self-signed cert + [System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy + [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Ssl3, [Net.SecurityProtocolType]::Tls, [Net.SecurityProtocolType]::Tls11, [Net.SecurityProtocolType]::Tls12 $response = Invoke-WebRequest -Uri https://$NODE_IP:10250/pods -Headers @{'Authorization' = "Bearer $(Get-Content /var/run/secrets/kubernetes.io/serviceaccount/token)" } -UseBasicParsing if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { Write-Host "Response of the Invoke-WebRequest -uri https://$NODE_IP:10250/pods is : $($response.StatusCode)" From 84afb010d2a7dd417d5ef89e66a37ca2e0de925c Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 14:09:52 -0700 Subject: [PATCH 04/20] wip --- kubernetes/windows/main.ps1 | 106 +++++++++++++++++------------------- 1 file changed, 51 insertions(+), 55 deletions(-) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 0bda34d7b..ac0b9264f 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -1,14 +1,3 @@ -add-type @" -using System.Net; -using System.Security.Cryptography.X509Certificates; -public class TrustAllCertsPolicy : ICertificatePolicy { - public bool CheckValidationResult( - ServicePoint srvPoint, X509Certificate certificate, - WebRequest request, int certificateProblem) { - return true; - } -} -"@ function Confirm-WindowsServiceExists($name) { if (Get-Service $name -ErrorAction SilentlyContinue) { return $true @@ -122,63 +111,70 @@ function Set-EnvironmentVariables { function Get-ContainerRuntime { $containerRuntime = "docker" $NODE_IP = "" - if (![string]::IsNullOrEmpty([System.Environment]::GetEnvironmentVariable("NODE_IP", "PROCESS"))) { - $NODE_IP = [System.Environment]::GetEnvironmentVariable("NODE_IP", "PROCESS") - } - elseif (![string]::IsNullOrEmpty([System.Environment]::GetEnvironmentVariable("NODE_IP", "USER"))) { - $NODE_IP = [System.Environment]::GetEnvironmentVariable("NODE_IP", "USER") - } - elseif (![string]::IsNullOrEmpty([System.Environment]::GetEnvironmentVariable("NODE_IP", "MACHINE"))) { - $NODE_IP = [System.Environment]::GetEnvironmentVariable("NODE_IP", "MACHINE") - } + try { + if (![string]::IsNullOrEmpty([System.Environment]::GetEnvironmentVariable("NODE_IP", "PROCESS"))) { + $NODE_IP = [System.Environment]::GetEnvironmentVariable("NODE_IP", "PROCESS") + } + elseif (![string]::IsNullOrEmpty([System.Environment]::GetEnvironmentVariable("NODE_IP", "USER"))) { + $NODE_IP = [System.Environment]::GetEnvironmentVariable("NODE_IP", "USER") + } + elseif (![string]::IsNullOrEmpty([System.Environment]::GetEnvironmentVariable("NODE_IP", "MACHINE"))) { + $NODE_IP = [System.Environment]::GetEnvironmentVariable("NODE_IP", "MACHINE") + } - if (![string]::IsNullOrEmpty($NODE_IP)) { - Write-Host "Value of NODE_IP environment variable : $($NODE_IP)" - $response = Invoke-WebRequest -uri http://$NODE_IP:10255/pods -UseBasicParsing - $isPodsAPISuccess = $false + if (![string]::IsNullOrEmpty($NODE_IP)) { + Write-Host "Value of NODE_IP environment variable : $($NODE_IP)" + $response = Invoke-WebRequest -uri http://$NODE_IP:10255/pods -UseBasicParsing + $isPodsAPISuccess = $false - if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { - Write-Host "Response of the Invoke-WebRequest -uri http://$NODE_IP:10255/pods is : $($response.StatusCode)" - $isPodsAPISuccess = $true - } - else { - # set the certificate policy to ignore the certificate policy since kubelet uses self-signed cert - [System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy - [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Ssl3, [Net.SecurityProtocolType]::Tls, [Net.SecurityProtocolType]::Tls11, [Net.SecurityProtocolType]::Tls12 - $response = Invoke-WebRequest -Uri https://$NODE_IP:10250/pods -Headers @{'Authorization' = "Bearer $(Get-Content /var/run/secrets/kubernetes.io/serviceaccount/token)" } -UseBasicParsing if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { - Write-Host "Response of the Invoke-WebRequest -uri https://$NODE_IP:10250/pods is : $($response.StatusCode)" + Write-Host "Response of the Invoke-WebRequest -uri http://$NODE_IP:10255/pods is : $($response.StatusCode)" $isPodsAPISuccess = $true } - } + else { + # set the certificate policy to ignore the certificate validation since kubelet uses self-signed cert + # [System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy + [System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true } + $response = Invoke-WebRequest -Uri https://$NODE_IP:10250/pods -Headers @{'Authorization' = "Bearer $(Get-Content /var/run/secrets/kubernetes.io/serviceaccount/token)" } -UseBasicParsing + if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { + Write-Host "Response of the Invoke-WebRequest -uri https://$NODE_IP:10250/pods is : $($response.StatusCode)" + $isPodsAPISuccess = $true + } + } - if ($isPodsAPISuccess -and ![string]::IsNullOrEmpty($response.Content)) { - $podList = $response.Content | ConvertFrom-Json - if (![string]::IsNullOrEmpty($podList)) { - $podItems = $podList.Items - if (![string]::IsNullOrEmpty($podItems) -and $podItems.Length -gt 0) { - Write-Host "found pod items: $($podItems.Length)" - for ($index = 0; $index -le $podItems.Length ; $index++) { - Write-Host "podItem index : $($index)" - $pod = $podItems[$index] - if (![string]::IsNullOrEmpty($pod) -and - ![string]::IsNullOrEmpty($pod.status) -and - ![string]::IsNullOrEmpty($pod.status.phase) -and - $pod.status.phase -eq "Running" -and - $pod.status.ContainerStatuses.Length -gt 0) { - $containerID = $pod.status.ContainerStatuses[0].containerID - $detectedContainerRuntime = $containerID.split(":")[0].trim() - if (![string]::IsNullOrEmpty($detectedContainerRuntime) -and [string]$detectedContainerRuntime.StartsWith('docker') -eq $false) { - $containerRuntime = $detectedContainerRuntime - Write-Host "detected containerRuntime as : $($containerRuntime)" + if ($isPodsAPISuccess -and ![string]::IsNullOrEmpty($response.Content)) { + $podList = $response.Content | ConvertFrom-Json + if (![string]::IsNullOrEmpty($podList)) { + $podItems = $podList.Items + if (![string]::IsNullOrEmpty($podItems) -and $podItems.Length -gt 0) { + Write-Host "found pod items: $($podItems.Length)" + for ($index = 0; $index -le $podItems.Length ; $index++) { + Write-Host "podItem index : $($index)" + $pod = $podItems[$index] + if (![string]::IsNullOrEmpty($pod) -and + ![string]::IsNullOrEmpty($pod.status) -and + ![string]::IsNullOrEmpty($pod.status.phase) -and + $pod.status.phase -eq "Running" -and + $pod.status.ContainerStatuses.Length -gt 0) { + $containerID = $pod.status.ContainerStatuses[0].containerID + $detectedContainerRuntime = $containerID.split(":")[0].trim() + if (![string]::IsNullOrEmpty($detectedContainerRuntime) -and [string]$detectedContainerRuntime.StartsWith('docker') -eq $false) { + $containerRuntime = $detectedContainerRuntime + Write-Host "detected containerRuntime as : $($containerRuntime)" + } + break } - break } } } } } } + catch { + $e = $_.Exception + Write-Host $e + Write-Host "exception occured on getting container runtime hence using default container runtime: $($containerRuntime)" + } return $containerRuntime } From 67b4584d67d7bdf2df234148d04fa35e521c4a90 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 16:06:29 -0700 Subject: [PATCH 05/20] bug fix related to uri --- kubernetes/windows/main.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index ac0b9264f..895d70f2a 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -124,20 +124,20 @@ function Get-ContainerRuntime { if (![string]::IsNullOrEmpty($NODE_IP)) { Write-Host "Value of NODE_IP environment variable : $($NODE_IP)" - $response = Invoke-WebRequest -uri http://$NODE_IP:10255/pods -UseBasicParsing + $response = Invoke-WebRequest -uri http://$($NODE_IP):10255/pods -UseBasicParsing $isPodsAPISuccess = $false if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { - Write-Host "Response of the Invoke-WebRequest -uri http://$NODE_IP:10255/pods is : $($response.StatusCode)" + Write-Host "Response of the Invoke-WebRequest -uri http://$($NODE_IP):10255/pods is : $($response.StatusCode)" $isPodsAPISuccess = $true } else { # set the certificate policy to ignore the certificate validation since kubelet uses self-signed cert # [System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy [System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true } - $response = Invoke-WebRequest -Uri https://$NODE_IP:10250/pods -Headers @{'Authorization' = "Bearer $(Get-Content /var/run/secrets/kubernetes.io/serviceaccount/token)" } -UseBasicParsing + $response = Invoke-WebRequest -Uri https://$($NODE_IP):10250/pods -Headers @{'Authorization' = "Bearer $(Get-Content /var/run/secrets/kubernetes.io/serviceaccount/token)" } -UseBasicParsing if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { - Write-Host "Response of the Invoke-WebRequest -uri https://$NODE_IP:10250/pods is : $($response.StatusCode)" + Write-Host "Response of the Invoke-WebRequest -uri https://$($NODE_IP):10250/pods is : $($response.StatusCode)" $isPodsAPISuccess = $true } } From 88e68879144e418b12e830d5097a09d28d449143 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 16:41:37 -0700 Subject: [PATCH 06/20] wip --- kubernetes/windows/main.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 895d70f2a..ced43b4fe 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -158,10 +158,11 @@ function Get-ContainerRuntime { $pod.status.ContainerStatuses.Length -gt 0) { $containerID = $pod.status.ContainerStatuses[0].containerID $detectedContainerRuntime = $containerID.split(":")[0].trim() + Write-Host "detected containerRuntime as : $($containerRuntime)" if (![string]::IsNullOrEmpty($detectedContainerRuntime) -and [string]$detectedContainerRuntime.StartsWith('docker') -eq $false) { $containerRuntime = $detectedContainerRuntime - Write-Host "detected containerRuntime as : $($containerRuntime)" } + Write-Host "using containerRuntime as : $($containerRuntime)" break } } From e5c81beef82d74635d37c10bc2e8414f79ca24dc Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 16:47:26 -0700 Subject: [PATCH 07/20] wip --- kubernetes/windows/main.ps1 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index ced43b4fe..d53d92262 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -170,6 +170,9 @@ function Get-ContainerRuntime { } } } + # set CONTAINER_RUNTIME env for debug and telemetry purpose + [System.Environment]::SetEnvironmentVariable("CONTAINER_RUNTIME", $containerRuntime, "Process") + [System.Environment]::SetEnvironmentVariable("CONTAINER_RUNTIME", $containerRuntime, "Machine") } catch { $e = $_.Exception From 3b4fe37d3227873a078b63ff74fa7172043ed247 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 17:33:23 -0700 Subject: [PATCH 08/20] fix bug with ignore cert validation --- kubernetes/windows/main.ps1 | 49 ++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 12 deletions(-) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index d53d92262..69a1a882e 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -1,3 +1,14 @@ +add-type @" +using System.Net; +using System.Security.Cryptography.X509Certificates; +public class TrustAllCertsPolicy : ICertificatePolicy { + public bool CheckValidationResult( + ServicePoint srvPoint, X509Certificate certificate, + WebRequest request, int certificateProblem) { + return true; + } +} +"@ function Confirm-WindowsServiceExists($name) { if (Get-Service $name -ErrorAction SilentlyContinue) { return $true @@ -110,6 +121,7 @@ function Set-EnvironmentVariables { function Get-ContainerRuntime { $containerRuntime = "docker" + $response = "" $NODE_IP = "" try { if (![string]::IsNullOrEmpty([System.Environment]::GetEnvironmentVariable("NODE_IP", "PROCESS"))) { @@ -123,23 +135,36 @@ function Get-ContainerRuntime { } if (![string]::IsNullOrEmpty($NODE_IP)) { - Write-Host "Value of NODE_IP environment variable : $($NODE_IP)" - $response = Invoke-WebRequest -uri http://$($NODE_IP):10255/pods -UseBasicParsing $isPodsAPISuccess = $false + Write-Host "Value of NODE_IP environment variable : $($NODE_IP)" + try { + Write-Host "Making API call to http://$($NODE_IP):10255/pods" + $response = Invoke-WebRequest -uri http://$($NODE_IP):10255/pods -UseBasicParsing + Write-Host "Response status code of API call to http://$($NODE_IP):10255/pods : $($response.StatusCode)" + } + catch { + Write-Host "API call to http://$($NODE_IP):10255/pods failed" + } if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { - Write-Host "Response of the Invoke-WebRequest -uri http://$($NODE_IP):10255/pods is : $($response.StatusCode)" + Write-Host "API call to http://$($NODE_IP):10255/pods succeeded" $isPodsAPISuccess = $true } else { - # set the certificate policy to ignore the certificate validation since kubelet uses self-signed cert - # [System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy - [System.Net.ServicePointManager]::ServerCertificateValidationCallback = { $true } - $response = Invoke-WebRequest -Uri https://$($NODE_IP):10250/pods -Headers @{'Authorization' = "Bearer $(Get-Content /var/run/secrets/kubernetes.io/serviceaccount/token)" } -UseBasicParsing - if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { - Write-Host "Response of the Invoke-WebRequest -uri https://$($NODE_IP):10250/pods is : $($response.StatusCode)" - $isPodsAPISuccess = $true - } + try { + Write-Host "Making API call to https://$($NODE_IP):10250/pods" + # set the certificate policy to ignore certificate validation since kubelet uses self-signed cert + [System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy + $response = Invoke-WebRequest -Uri https://$($NODE_IP):10250/pods -Headers @{'Authorization' = "Bearer $(Get-Content /var/run/secrets/kubernetes.io/serviceaccount/token)" } -UseBasicParsing + Write-Host "Response status code of API call to https://$($NODE_IP):10250/pods : $($response.StatusCode)" + if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { + Write-Host "API call to https://$($NODE_IP):10250/pods succeeded" + $isPodsAPISuccess = $true + } + } + catch { + Write-Host "API call to https://$($NODE_IP):10250/pods failed" + } } if ($isPodsAPISuccess -and ![string]::IsNullOrEmpty($response.Content)) { @@ -149,7 +174,7 @@ function Get-ContainerRuntime { if (![string]::IsNullOrEmpty($podItems) -and $podItems.Length -gt 0) { Write-Host "found pod items: $($podItems.Length)" for ($index = 0; $index -le $podItems.Length ; $index++) { - Write-Host "podItem index : $($index)" + Write-Host "current podItem index : $($index)" $pod = $podItems[$index] if (![string]::IsNullOrEmpty($pod) -and ![string]::IsNullOrEmpty($pod.status) -and From ac18f6a2d700e789bc7d862d32ffc95d03ddd424 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 19:09:12 -0700 Subject: [PATCH 09/20] logic to ignore cert validation --- kubernetes/windows/main.ps1 | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 69a1a882e..45b591e56 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -1,13 +1,25 @@ -add-type @" -using System.Net; -using System.Security.Cryptography.X509Certificates; -public class TrustAllCertsPolicy : ICertificatePolicy { - public bool CheckValidationResult( - ServicePoint srvPoint, X509Certificate certificate, - WebRequest request, int certificateProblem) { - return true; +Add-Type @" + using System; + using System.Net; + using System.Net.Security; + using System.Security.Cryptography.X509Certificates; + public class ServerCertificateValidationCallback + { + public static void Ignore() + { + ServicePointManager.ServerCertificateValidationCallback += + delegate + ( + Object obj, + X509Certificate certificate, + X509Chain chain, + SslPolicyErrors errors + ) + { + return true; + }; + } } -} "@ function Confirm-WindowsServiceExists($name) { if (Get-Service $name -ErrorAction SilentlyContinue) { @@ -153,8 +165,8 @@ function Get-ContainerRuntime { else { try { Write-Host "Making API call to https://$($NODE_IP):10250/pods" - # set the certificate policy to ignore certificate validation since kubelet uses self-signed cert - [System.Net.ServicePointManager]::CertificatePolicy = New-Object TrustAllCertsPolicy + # ignore certificate validation since kubelet uses self-signed cert + [ServerCertificateValidationCallback]::Ignore(); $response = Invoke-WebRequest -Uri https://$($NODE_IP):10250/pods -Headers @{'Authorization' = "Bearer $(Get-Content /var/run/secrets/kubernetes.io/serviceaccount/token)" } -UseBasicParsing Write-Host "Response status code of API call to https://$($NODE_IP):10250/pods : $($response.StatusCode)" if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { From 56d4e3ef8d97b7c348b23b56a905b8c5c912d9f8 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 19:24:59 -0700 Subject: [PATCH 10/20] minor --- kubernetes/windows/main.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 45b591e56..28436810b 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -166,7 +166,7 @@ function Get-ContainerRuntime { try { Write-Host "Making API call to https://$($NODE_IP):10250/pods" # ignore certificate validation since kubelet uses self-signed cert - [ServerCertificateValidationCallback]::Ignore(); + [ServerCertificateValidationCallback]::Ignore() $response = Invoke-WebRequest -Uri https://$($NODE_IP):10250/pods -Headers @{'Authorization' = "Bearer $(Get-Content /var/run/secrets/kubernetes.io/serviceaccount/token)" } -UseBasicParsing Write-Host "Response status code of API call to https://$($NODE_IP):10250/pods : $($response.StatusCode)" if (![string]::IsNullOrEmpty($response) -and $response.StatusCode -eq 200) { From c6edec6f2a461ba77bb22bd32d6e8ddcfbac4bb7 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 20:19:38 -0700 Subject: [PATCH 11/20] fix minor debug log issue --- kubernetes/windows/main.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 28436810b..0ab7fd762 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -195,7 +195,7 @@ function Get-ContainerRuntime { $pod.status.ContainerStatuses.Length -gt 0) { $containerID = $pod.status.ContainerStatuses[0].containerID $detectedContainerRuntime = $containerID.split(":")[0].trim() - Write-Host "detected containerRuntime as : $($containerRuntime)" + Write-Host "detected containerRuntime as : $($detectedContainerRuntime)" if (![string]::IsNullOrEmpty($detectedContainerRuntime) -and [string]$detectedContainerRuntime.StartsWith('docker') -eq $false) { $containerRuntime = $detectedContainerRuntime } From 5f06422bfd5bdc133ffe5a1169de9c703c6a815a Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 21:31:04 -0700 Subject: [PATCH 12/20] improve log message --- kubernetes/windows/main.ps1 | 76 +++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 28 deletions(-) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 0ab7fd762..73b3343ce 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -132,6 +132,7 @@ function Set-EnvironmentVariables { } function Get-ContainerRuntime { + # default container runtime and make default as containerd when containerd becomes default in AKS $containerRuntime = "docker" $response = "" $NODE_IP = "" @@ -150,9 +151,9 @@ function Get-ContainerRuntime { $isPodsAPISuccess = $false Write-Host "Value of NODE_IP environment variable : $($NODE_IP)" try { - Write-Host "Making API call to http://$($NODE_IP):10255/pods" - $response = Invoke-WebRequest -uri http://$($NODE_IP):10255/pods -UseBasicParsing - Write-Host "Response status code of API call to http://$($NODE_IP):10255/pods : $($response.StatusCode)" + Write-Host "Making API call to http://$($NODE_IP):10255/pods" + $response = Invoke-WebRequest -uri http://$($NODE_IP):10255/pods -UseBasicParsing + Write-Host "Response status code of API call to http://$($NODE_IP):10255/pods : $($response.StatusCode)" } catch { Write-Host "API call to http://$($NODE_IP):10255/pods failed" @@ -173,40 +174,59 @@ function Get-ContainerRuntime { Write-Host "API call to https://$($NODE_IP):10250/pods succeeded" $isPodsAPISuccess = $true } - } - catch { - Write-Host "API call to https://$($NODE_IP):10250/pods failed" - } + } + catch { + Write-Host "API call to https://$($NODE_IP):10250/pods failed" + } } - if ($isPodsAPISuccess -and ![string]::IsNullOrEmpty($response.Content)) { - $podList = $response.Content | ConvertFrom-Json - if (![string]::IsNullOrEmpty($podList)) { - $podItems = $podList.Items - if (![string]::IsNullOrEmpty($podItems) -and $podItems.Length -gt 0) { - Write-Host "found pod items: $($podItems.Length)" - for ($index = 0; $index -le $podItems.Length ; $index++) { - Write-Host "current podItem index : $($index)" - $pod = $podItems[$index] - if (![string]::IsNullOrEmpty($pod) -and - ![string]::IsNullOrEmpty($pod.status) -and - ![string]::IsNullOrEmpty($pod.status.phase) -and - $pod.status.phase -eq "Running" -and - $pod.status.ContainerStatuses.Length -gt 0) { - $containerID = $pod.status.ContainerStatuses[0].containerID - $detectedContainerRuntime = $containerID.split(":")[0].trim() - Write-Host "detected containerRuntime as : $($detectedContainerRuntime)" - if (![string]::IsNullOrEmpty($detectedContainerRuntime) -and [string]$detectedContainerRuntime.StartsWith('docker') -eq $false) { - $containerRuntime = $detectedContainerRuntime + if ($isPodsAPISuccess) { + if (![string]::IsNullOrEmpty($response.Content)) { + $podList = $response.Content | ConvertFrom-Json + if (![string]::IsNullOrEmpty($podList)) { + $podItems = $podList.Items + if (![string]::IsNullOrEmpty($podItems)) { + if ($podItems.Length -gt 0) { + Write-Host "found pod items: $($podItems.Length)" + for ($index = 0; $index -le $podItems.Length ; $index++) { + Write-Host "current podItem index : $($index)" + $pod = $podItems[$index] + if (![string]::IsNullOrEmpty($pod) -and + ![string]::IsNullOrEmpty($pod.status) -and + ![string]::IsNullOrEmpty($pod.status.phase) -and + $pod.status.phase -eq "Running" -and + $pod.status.ContainerStatuses.Length -gt 0) { + $containerID = $pod.status.ContainerStatuses[0].containerID + $detectedContainerRuntime = $containerID.split(":")[0].trim() + Write-Host "detected containerRuntime as : $($detectedContainerRuntime)" + if (![string]::IsNullOrEmpty($detectedContainerRuntime) -and [string]$detectedContainerRuntime.StartsWith('docker') -eq $false) { + $containerRuntime = $detectedContainerRuntime + } + Write-Host "using containerRuntime as : $($containerRuntime)" + break + } } - Write-Host "using containerRuntime as : $($containerRuntime)" - break } + else { + Write-Host "got podItems count is 0 hence using default container runtime: $($containerRuntime)" + } + } + else { + Write-Host "got podItems null or empty hence using default container runtime: $($containerRuntime)" } } + else { + Write-Host "got podList null or empty hence using default container runtime: $($containerRuntime)" + } + } + else { + Write-Host "got empty response content for /Pods API call hence using default container runtime: $($containerRuntime)" } } } + else { + Write-Host "got empty NODE_IP environment variable" + } # set CONTAINER_RUNTIME env for debug and telemetry purpose [System.Environment]::SetEnvironmentVariable("CONTAINER_RUNTIME", $containerRuntime, "Process") [System.Environment]::SetEnvironmentVariable("CONTAINER_RUNTIME", $containerRuntime, "Machine") From d635ce296d68b9ee7e550e6efbd05f80d97228e6 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 16 Aug 2020 22:23:45 -0700 Subject: [PATCH 13/20] debug message --- kubernetes/windows/main.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 73b3343ce..793ebc032 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -182,6 +182,7 @@ function Get-ContainerRuntime { if ($isPodsAPISuccess) { if (![string]::IsNullOrEmpty($response.Content)) { + Write-Host "response content: $($response.Content)" $podList = $response.Content | ConvertFrom-Json if (![string]::IsNullOrEmpty($podList)) { $podItems = $podList.Items From 439ba8ae5d331dfd6ce45df8c49e41d1c9f3e726 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 17 Aug 2020 12:25:49 -0700 Subject: [PATCH 14/20] fix bug with nullorempty check --- kubernetes/windows/main.ps1 | 43 +++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 793ebc032..9bb4c1a7e 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -186,35 +186,32 @@ function Get-ContainerRuntime { $podList = $response.Content | ConvertFrom-Json if (![string]::IsNullOrEmpty($podList)) { $podItems = $podList.Items - if (![string]::IsNullOrEmpty($podItems)) { - if ($podItems.Length -gt 0) { - Write-Host "found pod items: $($podItems.Length)" - for ($index = 0; $index -le $podItems.Length ; $index++) { - Write-Host "current podItem index : $($index)" - $pod = $podItems[$index] - if (![string]::IsNullOrEmpty($pod) -and - ![string]::IsNullOrEmpty($pod.status) -and - ![string]::IsNullOrEmpty($pod.status.phase) -and - $pod.status.phase -eq "Running" -and - $pod.status.ContainerStatuses.Length -gt 0) { - $containerID = $pod.status.ContainerStatuses[0].containerID - $detectedContainerRuntime = $containerID.split(":")[0].trim() - Write-Host "detected containerRuntime as : $($detectedContainerRuntime)" - if (![string]::IsNullOrEmpty($detectedContainerRuntime) -and [string]$detectedContainerRuntime.StartsWith('docker') -eq $false) { - $containerRuntime = $detectedContainerRuntime - } - Write-Host "using containerRuntime as : $($containerRuntime)" - break + if ($podItems.Length -gt 0) { + Write-Host "found pod items: $($podItems.Length)" + for ($index = 0; $index -le $podItems.Length ; $index++) { + Write-Host "current podItem index : $($index)" + $pod = $podItems[$index] + if (![string]::IsNullOrEmpty($pod) -and + ![string]::IsNullOrEmpty($pod.status) -and + ![string]::IsNullOrEmpty($pod.status.phase) -and + $pod.status.phase -eq "Running" -and + $pod.status.ContainerStatuses.Length -gt 0) { + $containerID = $pod.status.ContainerStatuses[0].containerID + $detectedContainerRuntime = $containerID.split(":")[0].trim() + Write-Host "detected containerRuntime as : $($detectedContainerRuntime)" + if (![string]::IsNullOrEmpty($detectedContainerRuntime) -and [string]$detectedContainerRuntime.StartsWith('docker') -eq $false) { + $containerRuntime = $detectedContainerRuntime } + Write-Host "using containerRuntime as : $($containerRuntime)" + break } } - else { - Write-Host "got podItems count is 0 hence using default container runtime: $($containerRuntime)" - } } else { - Write-Host "got podItems null or empty hence using default container runtime: $($containerRuntime)" + Write-Host "got podItems count is 0 hence using default container runtime: $($containerRuntime)" } + + } else { Write-Host "got podList null or empty hence using default container runtime: $($containerRuntime)" From c41962433634cd2b8654abcb700ff0ff793681bd Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 17 Aug 2020 14:05:49 -0700 Subject: [PATCH 15/20] remove debug statements --- kubernetes/windows/main.ps1 | 1 - 1 file changed, 1 deletion(-) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 9bb4c1a7e..08479a326 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -182,7 +182,6 @@ function Get-ContainerRuntime { if ($isPodsAPISuccess) { if (![string]::IsNullOrEmpty($response.Content)) { - Write-Host "response content: $($response.Content)" $podList = $response.Content | ConvertFrom-Json if (![string]::IsNullOrEmpty($podList)) { $podItems = $podList.Items From f7b503f3d45c5112cd7ba409a645f4c196bce5eb Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 17 Aug 2020 15:37:51 -0700 Subject: [PATCH 16/20] refactor parsers --- .../installer/conf/fluent-cri-parser.conf | 6 ++ build/windows/installer/conf/fluent-cri.conf | 73 ------------------- .../installer/conf/fluent-docker-parser.conf | 5 ++ build/windows/installer/conf/fluent.conf | 14 +--- kubernetes/windows/Dockerfile | 5 +- kubernetes/windows/main.ps1 | 8 +- 6 files changed, 22 insertions(+), 89 deletions(-) create mode 100644 build/windows/installer/conf/fluent-cri-parser.conf delete mode 100644 build/windows/installer/conf/fluent-cri.conf create mode 100644 build/windows/installer/conf/fluent-docker-parser.conf diff --git a/build/windows/installer/conf/fluent-cri-parser.conf b/build/windows/installer/conf/fluent-cri-parser.conf new file mode 100644 index 000000000..86f1572ca --- /dev/null +++ b/build/windows/installer/conf/fluent-cri-parser.conf @@ -0,0 +1,6 @@ + + @type regexp + expression ^(? diff --git a/build/windows/installer/conf/fluent-cri.conf b/build/windows/installer/conf/fluent-cri.conf deleted file mode 100644 index 5919110dc..000000000 --- a/build/windows/installer/conf/fluent-cri.conf +++ /dev/null @@ -1,73 +0,0 @@ - - type heartbeat_request - run_interval 30m - @log_level info - - - - @type tail - path /var/log/containers/*.log - pos_file /var/opt/microsoft/fluent/fluentd-containers.log.pos - tag oms.container.log.la - @log_level trace - path_key tailed_path - limit_recently_modified 5m - - @type regexp - expression ^(? - - - - @type tail - path /var/log/containers/omsagent*.log - pos_file /opt/microsoft/fluent/omsagent-fluentd-containers.log.pos - tag oms.container.log.flbplugin - @log_level trace - path_key tailed_path - read_from_head true - - @type regexp - expression ^(? - - - - @type record_transformer - # fluent-plugin-record-modifier more light-weight but needs to be installed (dependency worth it?) - remove_keys tailed_path - - filepath ${record["tailed_path"]} - - - - - - @type forward - send_timeout 60s - recover_wait 10s - hard_timeout 60s - heartbeat_type none - ignore_network_errors_at_startup true - - name logaggregationserver - host 127.0.0.1 - port 25230 - weight 60 - - - - overflow_action throw_exception - chunk_limit_size 32k - queued_chunks_limit_size 256 - flush_interval 1 - flush_thread_interval 0.5 - flush_thread_burst_interval 0.01 - flush_thread_count 4 - retry_forever true - - diff --git a/build/windows/installer/conf/fluent-docker-parser.conf b/build/windows/installer/conf/fluent-docker-parser.conf new file mode 100644 index 000000000..94727bfdc --- /dev/null +++ b/build/windows/installer/conf/fluent-docker-parser.conf @@ -0,0 +1,5 @@ + + @type json + time_format %Y-%m-%dT%H:%M:%S.%NZ + keep_time_key true + \ No newline at end of file diff --git a/build/windows/installer/conf/fluent.conf b/build/windows/installer/conf/fluent.conf index a4cacbcf6..327f226bb 100644 --- a/build/windows/installer/conf/fluent.conf +++ b/build/windows/installer/conf/fluent.conf @@ -12,11 +12,8 @@ @log_level trace path_key tailed_path limit_recently_modified 5m - - @type json - time_format %Y-%m-%dT%H:%M:%S.%NZ - keep_time_key true - + # if the container runtime is not docker then this will be updated with fluent-cri-parser.conf + @include fluent-docker-parser.conf @@ -27,11 +24,8 @@ @log_level trace path_key tailed_path read_from_head true - - @type json - time_format %Y-%m-%dT%H:%M:%S.%NZ - keep_time_key true - + # if the container runtime is not docker then this will be updated with fluent-cri-parser.conf + @include fluent-docker-parser.conf diff --git a/kubernetes/windows/Dockerfile b/kubernetes/windows/Dockerfile index 7979e8a3a..06e11e73a 100644 --- a/kubernetes/windows/Dockerfile +++ b/kubernetes/windows/Dockerfile @@ -56,8 +56,9 @@ COPY ./omsagentwindows/out_oms.so /opt/omsagentwindows/out_oms.so # copy fluent, fluent-bit and out_oms conf files COPY ./omsagentwindows/installer/conf/fluent.conf /etc/fluent/ -# copy fluent cri runtime conf file -COPY ./omsagentwindows/installer/conf/fluent-cri.conf /etc/fluent/ +# copy fluent docker and cri parser conf files +COPY ./omsagentwindows/installer/conf/fluent-cri-parser.conf /etc/fluent/ +COPY ./omsagentwindows/installer/conf/fluent-docker-parser.conf /etc/fluent/ COPY ./omsagentwindows/installer/conf/fluent-bit.conf /etc/fluent-bit COPY ./omsagentwindows/installer/conf/out_oms.conf /etc/omsagentwindows diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 08479a326..4054560c4 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -248,12 +248,12 @@ function Start-Fluent { #register fluentd as a service and start # there is a known issues with win32-service https://github.com/chef/win32-service/issues/70 if (![string]::IsNullOrEmpty($containerRuntime) -and [string]$containerRuntime.StartsWith('docker') -eq $false) { - fluentd --reg-winsvc i --reg-winsvc-auto-start --winsvc-name fluentdwinaks --reg-winsvc-fluentdopt '-c C:/etc/fluent/fluent-cri.conf -o C:/etc/fluent/fluent.log' - } - else { - fluentd --reg-winsvc i --reg-winsvc-auto-start --winsvc-name fluentdwinaks --reg-winsvc-fluentdopt '-c C:/etc/fluent/fluent.conf -o C:/etc/fluent/fluent.log' + # change parser from docker to cri if the container runtime is not docker + (Get-Content -Path C:/etc/fluent/fluent.conf -Raw) -replace 'fluent-docker-parser.conf','fluent-cri-parser.conf' | Set-Content C:/etc/fluent/fluent.conf } + fluentd --reg-winsvc i --reg-winsvc-auto-start --winsvc-name fluentdwinaks --reg-winsvc-fluentdopt '-c C:/etc/fluent/fluent.conf -o C:/etc/fluent/fluent.log' + Notepad.exe | Out-Null } From a97bd7388778dbb962ed620da838dda6b6b760c1 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 17 Aug 2020 16:44:35 -0700 Subject: [PATCH 17/20] add debug message --- kubernetes/windows/main.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index 4054560c4..de82722ad 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -249,6 +249,7 @@ function Start-Fluent { # there is a known issues with win32-service https://github.com/chef/win32-service/issues/70 if (![string]::IsNullOrEmpty($containerRuntime) -and [string]$containerRuntime.StartsWith('docker') -eq $false) { # change parser from docker to cri if the container runtime is not docker + Write-Host "changing parser from Docker to CRI since container runtime : $($containerRuntime) and which is non-docker" (Get-Content -Path C:/etc/fluent/fluent.conf -Raw) -replace 'fluent-docker-parser.conf','fluent-cri-parser.conf' | Set-Content C:/etc/fluent/fluent.conf } From c544cf0db4d809508bb792dba065062d8e53d5df Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 17 Aug 2020 16:47:57 -0700 Subject: [PATCH 18/20] clean up --- build/windows/installer/conf/fluent-docker-parser.conf | 2 +- build/windows/installer/conf/fluent.conf | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build/windows/installer/conf/fluent-docker-parser.conf b/build/windows/installer/conf/fluent-docker-parser.conf index 94727bfdc..66abc6026 100644 --- a/build/windows/installer/conf/fluent-docker-parser.conf +++ b/build/windows/installer/conf/fluent-docker-parser.conf @@ -2,4 +2,4 @@ @type json time_format %Y-%m-%dT%H:%M:%S.%NZ keep_time_key true - \ No newline at end of file + diff --git a/build/windows/installer/conf/fluent.conf b/build/windows/installer/conf/fluent.conf index 327f226bb..8e51a1569 100644 --- a/build/windows/installer/conf/fluent.conf +++ b/build/windows/installer/conf/fluent.conf @@ -12,7 +12,7 @@ @log_level trace path_key tailed_path limit_recently_modified 5m - # if the container runtime is not docker then this will be updated with fluent-cri-parser.conf + # if the container runtime is non docker then this will be updated to fluent-cri-parser.conf during container startup @include fluent-docker-parser.conf @@ -24,7 +24,7 @@ @log_level trace path_key tailed_path read_from_head true - # if the container runtime is not docker then this will be updated with fluent-cri-parser.conf + # if the container runtime is non docker then this will be updated to fluent-cri-parser.conf during container startup @include fluent-docker-parser.conf From 48184c9b698e8c2efdd67a017f172df890ecb836 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 17 Aug 2020 17:19:34 -0700 Subject: [PATCH 19/20] chart updates --- .../templates/omsagent-daemonset-windows.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml b/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml index 0ea7a9af6..b8e667398 100644 --- a/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml +++ b/charts/azuremonitor-containers/templates/omsagent-daemonset-windows.yaml @@ -53,6 +53,13 @@ spec: - name: CONTROLLER_TYPE value: "DaemonSet" - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP volumeMounts: - mountPath: C:\ProgramData\docker\containers name: docker-windows-containers From ee50b27b5d1e5dd915cd99c4d07b5cb21208b220 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Tue, 18 Aug 2020 11:40:32 -0700 Subject: [PATCH 20/20] fix formatting issues --- .../installer/conf/fluent-docker-parser.conf | 6 +++--- build/windows/installer/conf/fluent.conf | 18 +++++++++--------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/build/windows/installer/conf/fluent-docker-parser.conf b/build/windows/installer/conf/fluent-docker-parser.conf index 66abc6026..9dc800aeb 100644 --- a/build/windows/installer/conf/fluent-docker-parser.conf +++ b/build/windows/installer/conf/fluent-docker-parser.conf @@ -1,5 +1,5 @@ - @type json - time_format %Y-%m-%dT%H:%M:%S.%NZ - keep_time_key true + @type json + time_format %Y-%m-%dT%H:%M:%S.%NZ + keep_time_key true diff --git a/build/windows/installer/conf/fluent.conf b/build/windows/installer/conf/fluent.conf index 8e51a1569..c96300b1e 100644 --- a/build/windows/installer/conf/fluent.conf +++ b/build/windows/installer/conf/fluent.conf @@ -53,13 +53,13 @@ - overflow_action throw_exception - chunk_limit_size 32k - queued_chunks_limit_size 256 - flush_interval 1 - flush_thread_interval 0.5 - flush_thread_burst_interval 0.01 - flush_thread_count 4 - retry_forever true - + overflow_action throw_exception + chunk_limit_size 32k + queued_chunks_limit_size 256 + flush_interval 1 + flush_thread_interval 0.5 + flush_thread_burst_interval 0.01 + flush_thread_count 4 + retry_forever true +