From 981968c9a008c03c39b769b2ba2e0a92399eded6 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Tue, 31 Aug 2021 16:37:38 -0700 Subject: [PATCH 1/4] added script for collecting logs --- scripts/troubleshoot/collect_logs.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 scripts/troubleshoot/collect_logs.sh diff --git a/scripts/troubleshoot/collect_logs.sh b/scripts/troubleshoot/collect_logs.sh new file mode 100755 index 000000000..014659d5e --- /dev/null +++ b/scripts/troubleshoot/collect_logs.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +export ds_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-[a-z0-9]{5} | head -n 1) +export rs_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-rs-[a-z0-9]{5} | head -n 1) + +echo "collecting logs from ${ds_pod} and ${rs_pod}" + +kubectl cp ${ds_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-daemonset-log --namespace=kube-system + +kubectl cp ${ds_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-daemonset-mdsd-log --namespace=kube-system + +kubectl cp ${rs_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-replicaset-log --namespace=kube-system + +kubectl cp ${rs_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-replicaset-mdsd-log --namespace=kube-system + +zip -r azure-monitor-logs.zip omsagent-daemonset-log omsagent-daemonset-mdsd-log omsagent-replicaset-log omsagent-replicaset-mdsd-log + +rm -rf omsagent-daemonset-log omsagent-daemonset-mdsd-log omsagent-replicaset-log omsagent-replicaset-mdsd-log + +echo +echo "log files have been written to azure-monitor-logs.zip" From 606e01fabcc581e29828ac53eeb4c28f8541f615 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Tue, 31 Aug 2021 17:07:28 -0700 Subject: [PATCH 2/4] added windows daemonset and prometheus sidecar, as well as some explanatory prints --- scripts/troubleshoot/collect_logs.sh | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/scripts/troubleshoot/collect_logs.sh b/scripts/troubleshoot/collect_logs.sh index 014659d5e..4e59deaf1 100755 --- a/scripts/troubleshoot/collect_logs.sh +++ b/scripts/troubleshoot/collect_logs.sh @@ -1,21 +1,32 @@ #!/bin/bash + +# This script pulls logs from the replicaset agent pod and a random daemonset pod. This script is to make troubleshooting faster + export ds_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-[a-z0-9]{5} | head -n 1) +export ds_win_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-win-[a-z0-9]{5} | head -n 1) export rs_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-rs-[a-z0-9]{5} | head -n 1) -echo "collecting logs from ${ds_pod} and ${rs_pod}" +echo "collecting logs from ${ds_pod}, ${ds_win_pod}, and ${rs_pod}" +echo " note: some erros are expected for clusters without windows nodes, they can safely be disregarded (filespec must match the canonical format:, zip warning: name not matched: omsagent-win-daemonset-fbit)" -kubectl cp ${ds_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-daemonset-log --namespace=kube-system +kubectl cp ${ds_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-daemonset --namespace=kube-system --container omsagent +kubectl cp ${ds_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-daemonset-mdsd --namespace=kube-system --container omsagent -kubectl cp ${ds_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-daemonset-mdsd-log --namespace=kube-system +kubectl cp ${ds_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-prom-daemonset --namespace=kube-system --container omsagent-prometheus +kubectl cp ${ds_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-prom-daemonset-mdsd --namespace=kube-system --container omsagent-prometheus -kubectl cp ${rs_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-replicaset-log --namespace=kube-system +# for some reason copying logs out of /etc/omsagentwindows doesn't work (gives a permission error), but exec then cat does work. +# skip collecting these logs for now, would be good to come back and fix this next time a windows support case comes up +# kubectl cp ${ds_win_pod}:/etc/omsagentwindows omsagent-win-daemonset --namespace=kube-system +kubectl cp ${ds_win_pod}:/etc/fluent-bit omsagent-win-daemonset-fbit --namespace=kube-system -kubectl cp ${rs_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-replicaset-mdsd-log --namespace=kube-system +kubectl cp ${rs_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-replicaset --namespace=kube-system +kubectl cp ${rs_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-replicaset-mdsd --namespace=kube-system -zip -r azure-monitor-logs.zip omsagent-daemonset-log omsagent-daemonset-mdsd-log omsagent-replicaset-log omsagent-replicaset-mdsd-log +zip -r azure-monitor-logs.zip omsagent-daemonset omsagent-daemonset-mdsd omsagent-prom-daemonset omsagent-prom-daemonset-mdsd omsagent-win-daemonset-fbit omsagent-replicaset omsagent-replicaset-mdsd -rm -rf omsagent-daemonset-log omsagent-daemonset-mdsd-log omsagent-replicaset-log omsagent-replicaset-mdsd-log +rm -rf omsagent-daemonset omsagent-daemonset-mdsd omsagent-prom-daemonset omsagent-prom-daemonset-mdsd omsagent-win-daemonset-fbit omsagent-replicaset omsagent-replicaset-mdsd echo echo "log files have been written to azure-monitor-logs.zip" From a1845a5e3fabcfc099a48b553e215e0a49eaeebf Mon Sep 17 00:00:00 2001 From: David Michelman Date: Tue, 31 Aug 2021 17:27:31 -0700 Subject: [PATCH 3/4] added kubectl describe and kubectl logs output --- scripts/troubleshoot/collect_logs.sh | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/scripts/troubleshoot/collect_logs.sh b/scripts/troubleshoot/collect_logs.sh index 4e59deaf1..916d0defe 100755 --- a/scripts/troubleshoot/collect_logs.sh +++ b/scripts/troubleshoot/collect_logs.sh @@ -1,8 +1,10 @@ #!/bin/bash - # This script pulls logs from the replicaset agent pod and a random daemonset pod. This script is to make troubleshooting faster +mkdir azure-monitor-logs-tmp +cd azure-monitor-logs-tmp + export ds_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-[a-z0-9]{5} | head -n 1) export ds_win_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-win-[a-z0-9]{5} | head -n 1) export rs_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-rs-[a-z0-9]{5} | head -n 1) @@ -10,6 +12,23 @@ export rs_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata echo "collecting logs from ${ds_pod}, ${ds_win_pod}, and ${rs_pod}" echo " note: some erros are expected for clusters without windows nodes, they can safely be disregarded (filespec must match the canonical format:, zip warning: name not matched: omsagent-win-daemonset-fbit)" +# grab `kubectl describe` and `kubectl log` +echo "collecting kubectl describe and kubectl log output" + +kubectl describe pod ${ds_pod} --namespace=kube-system > describe_${ds_pod}.txt +kubectl logs ${ds_pod} --container omsagent --namespace=kube-system > logs_${ds_pod}.txt +kubectl logs ${ds_pod} --container omsagent-prometheus --namespace=kube-system > logs_${ds_pod}_prom.txt + +kubectl describe pod ${ds_win_pod} --namespace=kube-system > describe_${ds_win_pod}.txt +kubectl logs ${ds_win_pod} --container omsagent-win --namespace=kube-system > logs_${ds_win_pod}.txt + +kubectl describe pod ${rs_pod} --namespace=kube-system > describe_${rs_pod}.txt +kubectl logs ${rs_pod} --container omsagent --namespace=kube-system > logs_${rs_pod}.txt + + +# now collect log files from in containers +echo "collecting log files from inside agent containers" + kubectl cp ${ds_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-daemonset --namespace=kube-system --container omsagent kubectl cp ${ds_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-daemonset-mdsd --namespace=kube-system --container omsagent @@ -24,9 +43,9 @@ kubectl cp ${ds_win_pod}:/etc/fluent-bit omsagent-win-daemonset-fbit --namespace kubectl cp ${rs_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-replicaset --namespace=kube-system kubectl cp ${rs_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-replicaset-mdsd --namespace=kube-system -zip -r azure-monitor-logs.zip omsagent-daemonset omsagent-daemonset-mdsd omsagent-prom-daemonset omsagent-prom-daemonset-mdsd omsagent-win-daemonset-fbit omsagent-replicaset omsagent-replicaset-mdsd - -rm -rf omsagent-daemonset omsagent-daemonset-mdsd omsagent-prom-daemonset omsagent-prom-daemonset-mdsd omsagent-win-daemonset-fbit omsagent-replicaset omsagent-replicaset-mdsd +zip -r -q ../azure-monitor-logs.zip * +cd .. +rm -rf azure-monitor-logs-tmp echo echo "log files have been written to azure-monitor-logs.zip" From ef23941b89e1e260c147406b1e0ecc1c3a576053 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Tue, 31 Aug 2021 17:36:40 -0700 Subject: [PATCH 4/4] changed message to make it more clear some erros are expected --- scripts/troubleshoot/collect_logs.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/troubleshoot/collect_logs.sh b/scripts/troubleshoot/collect_logs.sh index 916d0defe..99a9ad302 100755 --- a/scripts/troubleshoot/collect_logs.sh +++ b/scripts/troubleshoot/collect_logs.sh @@ -2,6 +2,9 @@ # This script pulls logs from the replicaset agent pod and a random daemonset pod. This script is to make troubleshooting faster +CYAN='\033[0;36m' +NC='\033[0m' # No Color + mkdir azure-monitor-logs-tmp cd azure-monitor-logs-tmp @@ -9,8 +12,8 @@ export ds_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata export ds_win_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-win-[a-z0-9]{5} | head -n 1) export rs_pod=$(kubectl get pods -n kube-system -o custom-columns=NAME:.metadata.name | grep -E omsagent-rs-[a-z0-9]{5} | head -n 1) -echo "collecting logs from ${ds_pod}, ${ds_win_pod}, and ${rs_pod}" -echo " note: some erros are expected for clusters without windows nodes, they can safely be disregarded (filespec must match the canonical format:, zip warning: name not matched: omsagent-win-daemonset-fbit)" +echo -e "Collecting logs from ${ds_pod}, ${ds_win_pod}, and ${rs_pod}" +echo -e "${CYAN}Note: some errors about pods and files not existing are expected in clusters without windows nodes or sidecar prometheus scraping. They can safely be disregarded ${NC}" # grab `kubectl describe` and `kubectl log` echo "collecting kubectl describe and kubectl log output" @@ -27,7 +30,7 @@ kubectl logs ${rs_pod} --container omsagent --namespace=kube-system > logs_${rs_ # now collect log files from in containers -echo "collecting log files from inside agent containers" +echo "Collecting log files from inside agent containers" kubectl cp ${ds_pod}:/var/opt/microsoft/docker-cimprov/log omsagent-daemonset --namespace=kube-system --container omsagent kubectl cp ${ds_pod}:/var/opt/microsoft/linuxmonagent/log omsagent-daemonset-mdsd --namespace=kube-system --container omsagent