From 43355a6028d7c8a1fdf9753da21843949460c8e8 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Fri, 2 Jul 2021 13:42:58 -0700 Subject: [PATCH 1/6] linux build speed improvements --- kubernetes/linux/Dockerfile | 5 ++-- kubernetes/linux/setup.sh | 55 +++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/kubernetes/linux/Dockerfile b/kubernetes/linux/Dockerfile index 1ae7bef61..ceb7a8848 100644 --- a/kubernetes/linux/Dockerfile +++ b/kubernetes/linux/Dockerfile @@ -16,14 +16,15 @@ ENV AZMON_COLLECT_ENV False ENV KUBE_CLIENT_BACKOFF_BASE 1 ENV KUBE_CLIENT_BACKOFF_DURATION 0 ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9 -RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg && rm -rf /var/lib/apt/lists/* COPY setup.sh main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar mdsd.xml envmdsd $tmpdir/ WORKDIR ${tmpdir} +RUN chmod 775 $tmpdir/*.sh; sync; $tmpdir/setup.sh + # copy docker provider shell bundle to use the agent image COPY ./Linux_ULINUX_1.0_x64_64_Release/docker-cimprov-*.*.*-*.x86_64.sh . # Note: If you prefer remote destination, uncomment below line and comment above line # wget https://github.com/microsoft/Docker-Provider/releases/download/10.0.0-1/docker-cimprov-10.0.0-1.universal.x86_64.sh +RUN /bin/bash -c "/opt/docker-cimprov-*.*.*-*.x86_64.sh --install && rm -f /opt/docker-cimprov*.sh" -RUN chmod 775 $tmpdir/*.sh; sync; $tmpdir/setup.sh CMD [ "/opt/main.sh" ] diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index ad7cc2232..d4788350a 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -1,9 +1,26 @@ -TMPDIR="/opt" -cd $TMPDIR +cd /opt + +# install ruby2.6 +sudo apt-get install software-properties-common -y +sudo apt-add-repository ppa:brightbox/ruby-ng -y +#fluent-bit(td-agent-bit) +wget -qO - https://packages.fluentbit.io/fluentbit.key | sudo apt-key add - +sudo echo "deb https://packages.fluentbit.io/ubuntu/xenial xenial main" >> /etc/apt/sources.list +sudo apt-get update + +apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg + +# install telegraf in parallel with everything else to save build time +( + wget https://dl.influxdata.com/telegraf/releases/telegraf-1.18.0_linux_amd64.tar.gz + tar -zxvf telegraf-1.18.0_linux_amd64.tar.gz + mv /opt/telegraf-1.18.0/usr/bin/telegraf /opt/telegraf + chmod 777 /opt/telegraf +) & #Download utf-8 encoding capability on the omsagent container. #upgrade apt to latest version -apt-get update && apt-get install -y apt && DEBIAN_FRONTEND=noninteractive apt-get install -y locales +apt-get install -y apt && DEBIAN_FRONTEND=noninteractive apt-get install -y locales sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ @@ -12,12 +29,11 @@ sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ #install oneagent - Official bits (05/17/2021) wget https://github.com/microsoft/Docker-Provider/releases/download/05172021-oneagent/azure-mdsd_1.10.1-build.master.213_x86_64.deb -/usr/bin/dpkg -i $TMPDIR/azure-mdsd*.deb -cp -f $TMPDIR/mdsd.xml /etc/mdsd.d -cp -f $TMPDIR/envmdsd /etc/mdsd.d +/usr/bin/dpkg -i /opt/azure-mdsd*.deb +cp -f /opt/mdsd.xml /etc/mdsd.d +cp -f /opt/envmdsd /etc/mdsd.d #download inotify tools for watching configmap changes -sudo apt-get update sudo apt-get install inotify-tools -y #used to parse response of kubelet apis @@ -27,26 +43,10 @@ sudo apt-get install jq=1.5+dfsg-2 -y #used to setcaps for ruby process to read /proc/env sudo apt-get install libcap2-bin -y -wget https://dl.influxdata.com/telegraf/releases/telegraf-1.18.0_linux_amd64.tar.gz -tar -zxvf telegraf-1.18.0_linux_amd64.tar.gz - -mv /opt/telegraf-1.18.0/usr/bin/telegraf /opt/telegraf - -chmod 777 /opt/telegraf - -# Use wildcard version so that it doesnt require to touch this file -/$TMPDIR/docker-cimprov-*.*.*-*.x86_64.sh --install - #download and install fluent-bit(td-agent-bit) -wget -qO - https://packages.fluentbit.io/fluentbit.key | sudo apt-key add - -sudo echo "deb https://packages.fluentbit.io/ubuntu/xenial xenial main" >> /etc/apt/sources.list -sudo apt-get update sudo apt-get install td-agent-bit=1.6.8 -y # install ruby2.6 -sudo apt-get install software-properties-common -y -sudo apt-add-repository ppa:brightbox/ruby-ng -y -sudo apt-get update sudo apt-get install ruby2.6 ruby2.6-dev gcc make -y # fluentd v1 gem gem install fluentd -v "1.12.2" --no-document @@ -54,10 +54,9 @@ fluentd --setup ./fluent gem install gyoku iso8601 --no-doc -rm -f $TMPDIR/docker-cimprov*.sh -rm -f $TMPDIR/azure-mdsd*.deb -rm -f $TMPDIR/mdsd.xml -rm -f $TMPDIR/envmdsd +rm -f /opt/azure-mdsd*.deb +rm -f /opt/mdsd.xml +rm -f /opt/envmdsd # Remove settings for cron.daily that conflict with the node's cron.daily. Since both are trying to rotate the same files # in /var/log at the same time, the rotation doesn't happen correctly and then the *.1 file is forever logged to. @@ -70,3 +69,5 @@ if [ -e "/var/lib/gems/2.6.0/gems/http_parser.rb-0.6.0/Gemfile.lock" ]; then echo "Renaming unused gemfile.lock for http_parser 0.6.0" mv /var/lib/gems/2.6.0/gems/http_parser.rb-0.6.0/Gemfile.lock /var/lib/gems/2.6.0/gems/http_parser.rb-0.6.0/renamed_Gemfile_lock.renamed fi + +wait # in case telegraf hasn't finished installing yet \ No newline at end of file From fd61e505bef64814579f6a2d54c33110bb6f1582 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Fri, 2 Jul 2021 13:48:17 -0700 Subject: [PATCH 2/6] small changes --- kubernetes/linux/setup.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index d4788350a..059a166e7 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -1,11 +1,12 @@ cd /opt -# install ruby2.6 +# set up apt repo for ruby2.6 sudo apt-get install software-properties-common -y sudo apt-add-repository ppa:brightbox/ruby-ng -y -#fluent-bit(td-agent-bit) +# set up apt repo for fluent-bit(td-agent-bit) wget -qO - https://packages.fluentbit.io/fluentbit.key | sudo apt-key add - sudo echo "deb https://packages.fluentbit.io/ubuntu/xenial xenial main" >> /etc/apt/sources.list + sudo apt-get update apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg @@ -70,4 +71,4 @@ if [ -e "/var/lib/gems/2.6.0/gems/http_parser.rb-0.6.0/Gemfile.lock" ]; then mv /var/lib/gems/2.6.0/gems/http_parser.rb-0.6.0/Gemfile.lock /var/lib/gems/2.6.0/gems/http_parser.rb-0.6.0/renamed_Gemfile_lock.renamed fi -wait # in case telegraf hasn't finished installing yet \ No newline at end of file +wait # in case telegraf hasn't finished installing yet From f0f3cf7403db97a6f65f696753a6599f79ea7722 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Wed, 7 Jul 2021 16:54:32 -0700 Subject: [PATCH 3/6] added protection against node json with missing data --- source/plugins/ruby/in_kube_nodes.rb | 74 +++++++++++++++------------- 1 file changed, 40 insertions(+), 34 deletions(-) diff --git a/source/plugins/ruby/in_kube_nodes.rb b/source/plugins/ruby/in_kube_nodes.rb index ebfa903fd..5101876d2 100644 --- a/source/plugins/ruby/in_kube_nodes.rb +++ b/source/plugins/ruby/in_kube_nodes.rb @@ -280,48 +280,54 @@ def parse_and_emit_records(nodeInventory, batchTime = Time.utc.iso8601) timeDifference = (DateTime.now.to_time.to_i - @@nodeTelemetryTimeTracker).abs timeDifferenceInMinutes = timeDifference / 60 if (timeDifferenceInMinutes >= Constants::TELEMETRY_FLUSH_INTERVAL_IN_MINUTES) - properties = getNodeTelemetryProps(item) - properties["KubernetesProviderID"] = nodeInventoryRecord["KubernetesProviderID"] - capacityInfo = item["status"]["capacity"] - - ApplicationInsightsUtility.sendMetricTelemetry("NodeMemory", capacityInfo["memory"], properties) begin - if (!capacityInfo["nvidia.com/gpu"].nil?) && (!capacityInfo["nvidia.com/gpu"].empty?) - properties["nvigpus"] = capacityInfo["nvidia.com/gpu"] + properties = getNodeTelemetryProps(item) + properties["KubernetesProviderID"] = nodeInventoryRecord["KubernetesProviderID"] + capacityInfo = item["status"]["capacity"] + + ApplicationInsightsUtility.sendMetricTelemetry("NodeMemory", capacityInfo["memory"], properties) + begin + if (!capacityInfo["nvidia.com/gpu"].nil?) && (!capacityInfo["nvidia.com/gpu"].empty?) + properties["nvigpus"] = capacityInfo["nvidia.com/gpu"] + end + + if (!capacityInfo["amd.com/gpu"].nil?) && (!capacityInfo["amd.com/gpu"].empty?) + properties["amdgpus"] = capacityInfo["amd.com/gpu"] + end + rescue => errorStr + $log.warn "Failed in getting GPU telemetry in_kube_nodes : #{errorStr}" + $log.debug_backtrace(errorStr.backtrace) + ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) + end + + # Telemetry for data collection config for replicaset + if (File.file?(@@configMapMountPath)) + properties["collectAllKubeEvents"] = @@collectAllKubeEvents end - if (!capacityInfo["amd.com/gpu"].nil?) && (!capacityInfo["amd.com/gpu"].empty?) - properties["amdgpus"] = capacityInfo["amd.com/gpu"] + #telemetry about prometheus metric collections settings for replicaset + if (File.file?(@@promConfigMountPath)) + properties["rsPromInt"] = @@rsPromInterval + properties["rsPromFPC"] = @@rsPromFieldPassCount + properties["rsPromFDC"] = @@rsPromFieldDropCount + properties["rsPromServ"] = @@rsPromK8sServiceCount + properties["rsPromUrl"] = @@rsPromUrlCount + properties["rsPromMonPods"] = @@rsPromMonitorPods + properties["rsPromMonPodsNs"] = @@rsPromMonitorPodsNamespaceLength + properties["rsPromMonPodsLabelSelectorLength"] = @@rsPromMonitorPodsLabelSelectorLength + properties["rsPromMonPodsFieldSelectorLength"] = @@rsPromMonitorPodsFieldSelectorLength end + # telemetry about osm metric settings for replicaset + if (File.file?(@@osmConfigMountPath)) + properties["osmNamespaceCount"] = @@osmNamespaceCount + end + ApplicationInsightsUtility.sendMetricTelemetry("NodeCoreCapacity", capacityInfo["cpu"], properties) + telemetrySent = true rescue => errorStr - $log.warn "Failed in getting GPU telemetry in_kube_nodes : #{errorStr}" + $log.warn "Failed in getting telemetry in_kube_nodes : #{errorStr}" $log.debug_backtrace(errorStr.backtrace) ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) end - - # Telemetry for data collection config for replicaset - if (File.file?(@@configMapMountPath)) - properties["collectAllKubeEvents"] = @@collectAllKubeEvents - end - - #telemetry about prometheus metric collections settings for replicaset - if (File.file?(@@promConfigMountPath)) - properties["rsPromInt"] = @@rsPromInterval - properties["rsPromFPC"] = @@rsPromFieldPassCount - properties["rsPromFDC"] = @@rsPromFieldDropCount - properties["rsPromServ"] = @@rsPromK8sServiceCount - properties["rsPromUrl"] = @@rsPromUrlCount - properties["rsPromMonPods"] = @@rsPromMonitorPods - properties["rsPromMonPodsNs"] = @@rsPromMonitorPodsNamespaceLength - properties["rsPromMonPodsLabelSelectorLength"] = @@rsPromMonitorPodsLabelSelectorLength - properties["rsPromMonPodsFieldSelectorLength"] = @@rsPromMonitorPodsFieldSelectorLength - end - # telemetry about osm metric settings for replicaset - if (File.file?(@@osmConfigMountPath)) - properties["osmNamespaceCount"] = @@osmNamespaceCount - end - ApplicationInsightsUtility.sendMetricTelemetry("NodeCoreCapacity", capacityInfo["cpu"], properties) - telemetrySent = true end end if telemetrySent == true From e6a5e4b00f99dfaadb01a52042d2e30194b3d9b7 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Thu, 8 Jul 2021 16:28:35 -0700 Subject: [PATCH 4/6] fixed build speedups --- kubernetes/linux/Dockerfile | 6 +++--- kubernetes/linux/setup.sh | 10 +++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/kubernetes/linux/Dockerfile b/kubernetes/linux/Dockerfile index ceb7a8848..1b230f0cb 100644 --- a/kubernetes/linux/Dockerfile +++ b/kubernetes/linux/Dockerfile @@ -16,15 +16,15 @@ ENV AZMON_COLLECT_ENV False ENV KUBE_CLIENT_BACKOFF_BASE 1 ENV KUBE_CLIENT_BACKOFF_DURATION 0 ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9 -COPY setup.sh main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar mdsd.xml envmdsd $tmpdir/ +COPY setup.sh mdsd.xml envmdsd $tmpdir/ WORKDIR ${tmpdir} RUN chmod 775 $tmpdir/*.sh; sync; $tmpdir/setup.sh # copy docker provider shell bundle to use the agent image +COPY main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar $tmpdir/ COPY ./Linux_ULINUX_1.0_x64_64_Release/docker-cimprov-*.*.*-*.x86_64.sh . # Note: If you prefer remote destination, uncomment below line and comment above line # wget https://github.com/microsoft/Docker-Provider/releases/download/10.0.0-1/docker-cimprov-10.0.0-1.universal.x86_64.sh -RUN /bin/bash -c "/opt/docker-cimprov-*.*.*-*.x86_64.sh --install && rm -f /opt/docker-cimprov*.sh" - +RUN /bin/bash -c "/opt/docker-cimprov-*.*.*-*.x86_64.sh --install && rm -f /opt/docker-cimprov*.sh; chmod 775 $tmpdir/*.sh" CMD [ "/opt/main.sh" ] diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 059a166e7..bec74a69f 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -1,15 +1,19 @@ +set -e # exit if any command exits with an error + cd /opt +apt-get update +apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg + # set up apt repo for ruby2.6 sudo apt-get install software-properties-common -y sudo apt-add-repository ppa:brightbox/ruby-ng -y # set up apt repo for fluent-bit(td-agent-bit) wget -qO - https://packages.fluentbit.io/fluentbit.key | sudo apt-key add - -sudo echo "deb https://packages.fluentbit.io/ubuntu/xenial xenial main" >> /etc/apt/sources.list +echo "deb https://packages.fluentbit.io/ubuntu/xenial xenial main" >> /etc/apt/sources.list -sudo apt-get update +apt-get update -apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg # install telegraf in parallel with everything else to save build time ( From f9fd5705cc213e75fda1666382bc4a1e93636879 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Thu, 8 Jul 2021 16:29:21 -0700 Subject: [PATCH 5/6] Revert "fixed build speedups" This reverts commit e6a5e4b00f99dfaadb01a52042d2e30194b3d9b7. --- kubernetes/linux/Dockerfile | 6 +++--- kubernetes/linux/setup.sh | 10 +++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/kubernetes/linux/Dockerfile b/kubernetes/linux/Dockerfile index 1b230f0cb..ceb7a8848 100644 --- a/kubernetes/linux/Dockerfile +++ b/kubernetes/linux/Dockerfile @@ -16,15 +16,15 @@ ENV AZMON_COLLECT_ENV False ENV KUBE_CLIENT_BACKOFF_BASE 1 ENV KUBE_CLIENT_BACKOFF_DURATION 0 ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9 -COPY setup.sh mdsd.xml envmdsd $tmpdir/ +COPY setup.sh main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar mdsd.xml envmdsd $tmpdir/ WORKDIR ${tmpdir} RUN chmod 775 $tmpdir/*.sh; sync; $tmpdir/setup.sh # copy docker provider shell bundle to use the agent image -COPY main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar $tmpdir/ COPY ./Linux_ULINUX_1.0_x64_64_Release/docker-cimprov-*.*.*-*.x86_64.sh . # Note: If you prefer remote destination, uncomment below line and comment above line # wget https://github.com/microsoft/Docker-Provider/releases/download/10.0.0-1/docker-cimprov-10.0.0-1.universal.x86_64.sh -RUN /bin/bash -c "/opt/docker-cimprov-*.*.*-*.x86_64.sh --install && rm -f /opt/docker-cimprov*.sh; chmod 775 $tmpdir/*.sh" +RUN /bin/bash -c "/opt/docker-cimprov-*.*.*-*.x86_64.sh --install && rm -f /opt/docker-cimprov*.sh" + CMD [ "/opt/main.sh" ] diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index bec74a69f..059a166e7 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -1,19 +1,15 @@ -set -e # exit if any command exits with an error - cd /opt -apt-get update -apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg - # set up apt repo for ruby2.6 sudo apt-get install software-properties-common -y sudo apt-add-repository ppa:brightbox/ruby-ng -y # set up apt repo for fluent-bit(td-agent-bit) wget -qO - https://packages.fluentbit.io/fluentbit.key | sudo apt-key add - -echo "deb https://packages.fluentbit.io/ubuntu/xenial xenial main" >> /etc/apt/sources.list +sudo echo "deb https://packages.fluentbit.io/ubuntu/xenial xenial main" >> /etc/apt/sources.list -apt-get update +sudo apt-get update +apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg # install telegraf in parallel with everything else to save build time ( From f83ce831f7fae45e516b36a109b0a0ce5b6308e1 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Thu, 8 Jul 2021 16:30:27 -0700 Subject: [PATCH 6/6] Revert "Merge branch 'david/faster_builds' into david/malformed-node-fix" This reverts commit e54ecf9ea852f498d85187224c889a2a794724ca, reversing changes made to f0f3cf7403db97a6f65f696753a6599f79ea7722. --- kubernetes/linux/Dockerfile | 5 ++-- kubernetes/linux/setup.sh | 56 ++++++++++++++++++------------------- 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/kubernetes/linux/Dockerfile b/kubernetes/linux/Dockerfile index ceb7a8848..1ae7bef61 100644 --- a/kubernetes/linux/Dockerfile +++ b/kubernetes/linux/Dockerfile @@ -16,15 +16,14 @@ ENV AZMON_COLLECT_ENV False ENV KUBE_CLIENT_BACKOFF_BASE 1 ENV KUBE_CLIENT_BACKOFF_DURATION 0 ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9 +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg && rm -rf /var/lib/apt/lists/* COPY setup.sh main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar mdsd.xml envmdsd $tmpdir/ WORKDIR ${tmpdir} -RUN chmod 775 $tmpdir/*.sh; sync; $tmpdir/setup.sh - # copy docker provider shell bundle to use the agent image COPY ./Linux_ULINUX_1.0_x64_64_Release/docker-cimprov-*.*.*-*.x86_64.sh . # Note: If you prefer remote destination, uncomment below line and comment above line # wget https://github.com/microsoft/Docker-Provider/releases/download/10.0.0-1/docker-cimprov-10.0.0-1.universal.x86_64.sh -RUN /bin/bash -c "/opt/docker-cimprov-*.*.*-*.x86_64.sh --install && rm -f /opt/docker-cimprov*.sh" +RUN chmod 775 $tmpdir/*.sh; sync; $tmpdir/setup.sh CMD [ "/opt/main.sh" ] diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 059a166e7..ad7cc2232 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -1,27 +1,9 @@ -cd /opt - -# set up apt repo for ruby2.6 -sudo apt-get install software-properties-common -y -sudo apt-add-repository ppa:brightbox/ruby-ng -y -# set up apt repo for fluent-bit(td-agent-bit) -wget -qO - https://packages.fluentbit.io/fluentbit.key | sudo apt-key add - -sudo echo "deb https://packages.fluentbit.io/ubuntu/xenial xenial main" >> /etc/apt/sources.list - -sudo apt-get update - -apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg - -# install telegraf in parallel with everything else to save build time -( - wget https://dl.influxdata.com/telegraf/releases/telegraf-1.18.0_linux_amd64.tar.gz - tar -zxvf telegraf-1.18.0_linux_amd64.tar.gz - mv /opt/telegraf-1.18.0/usr/bin/telegraf /opt/telegraf - chmod 777 /opt/telegraf -) & +TMPDIR="/opt" +cd $TMPDIR #Download utf-8 encoding capability on the omsagent container. #upgrade apt to latest version -apt-get install -y apt && DEBIAN_FRONTEND=noninteractive apt-get install -y locales +apt-get update && apt-get install -y apt && DEBIAN_FRONTEND=noninteractive apt-get install -y locales sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ @@ -30,11 +12,12 @@ sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ #install oneagent - Official bits (05/17/2021) wget https://github.com/microsoft/Docker-Provider/releases/download/05172021-oneagent/azure-mdsd_1.10.1-build.master.213_x86_64.deb -/usr/bin/dpkg -i /opt/azure-mdsd*.deb -cp -f /opt/mdsd.xml /etc/mdsd.d -cp -f /opt/envmdsd /etc/mdsd.d +/usr/bin/dpkg -i $TMPDIR/azure-mdsd*.deb +cp -f $TMPDIR/mdsd.xml /etc/mdsd.d +cp -f $TMPDIR/envmdsd /etc/mdsd.d #download inotify tools for watching configmap changes +sudo apt-get update sudo apt-get install inotify-tools -y #used to parse response of kubelet apis @@ -44,10 +27,26 @@ sudo apt-get install jq=1.5+dfsg-2 -y #used to setcaps for ruby process to read /proc/env sudo apt-get install libcap2-bin -y +wget https://dl.influxdata.com/telegraf/releases/telegraf-1.18.0_linux_amd64.tar.gz +tar -zxvf telegraf-1.18.0_linux_amd64.tar.gz + +mv /opt/telegraf-1.18.0/usr/bin/telegraf /opt/telegraf + +chmod 777 /opt/telegraf + +# Use wildcard version so that it doesnt require to touch this file +/$TMPDIR/docker-cimprov-*.*.*-*.x86_64.sh --install + #download and install fluent-bit(td-agent-bit) +wget -qO - https://packages.fluentbit.io/fluentbit.key | sudo apt-key add - +sudo echo "deb https://packages.fluentbit.io/ubuntu/xenial xenial main" >> /etc/apt/sources.list +sudo apt-get update sudo apt-get install td-agent-bit=1.6.8 -y # install ruby2.6 +sudo apt-get install software-properties-common -y +sudo apt-add-repository ppa:brightbox/ruby-ng -y +sudo apt-get update sudo apt-get install ruby2.6 ruby2.6-dev gcc make -y # fluentd v1 gem gem install fluentd -v "1.12.2" --no-document @@ -55,9 +54,10 @@ fluentd --setup ./fluent gem install gyoku iso8601 --no-doc -rm -f /opt/azure-mdsd*.deb -rm -f /opt/mdsd.xml -rm -f /opt/envmdsd +rm -f $TMPDIR/docker-cimprov*.sh +rm -f $TMPDIR/azure-mdsd*.deb +rm -f $TMPDIR/mdsd.xml +rm -f $TMPDIR/envmdsd # Remove settings for cron.daily that conflict with the node's cron.daily. Since both are trying to rotate the same files # in /var/log at the same time, the rotation doesn't happen correctly and then the *.1 file is forever logged to. @@ -70,5 +70,3 @@ if [ -e "/var/lib/gems/2.6.0/gems/http_parser.rb-0.6.0/Gemfile.lock" ]; then echo "Renaming unused gemfile.lock for http_parser 0.6.0" mv /var/lib/gems/2.6.0/gems/http_parser.rb-0.6.0/Gemfile.lock /var/lib/gems/2.6.0/gems/http_parser.rb-0.6.0/renamed_Gemfile_lock.renamed fi - -wait # in case telegraf hasn't finished installing yet