From cacf15aee6910f29ef5e2d32b2bf57bc557c90dd Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 13 Jun 2021 22:58:38 -0700 Subject: [PATCH 01/14] changes related to aad msi auth feature --- .../installer/datafiles/base_container.data | 3 + kubernetes/linux/main.sh | 139 ++++-- kubernetes/linux/setup.sh | 2 +- kubernetes/omsagent.yaml | 7 + kubernetes/windows/main.ps1 | 24 + .../ci-extension-dcr-streams.md | 186 ++++++++ scripts/dcr-onboarding/ci-extension-dcr.json | 59 +++ source/plugins/go/src/extension/extension.go | 101 +++++ source/plugins/go/src/extension/interfaces.go | 34 ++ .../plugins/go/src/extension/socket_writer.go | 85 ++++ .../plugins/go/src/ingestion_token_utils.go | 429 ++++++++++++++++++ .../go/src/ingestion_token_utils_test.go | 54 +++ source/plugins/go/src/oms.go | 74 ++- source/plugins/go/src/utils.go | 43 +- .../ruby/ApplicationInsightsUtility.rb | 9 +- source/plugins/ruby/constants.rb | 23 + .../ruby/filter_health_model_builder.rb | 10 +- source/plugins/ruby/in_cadvisor_perf.rb | 14 +- source/plugins/ruby/in_containerinventory.rb | 10 +- source/plugins/ruby/in_kube_events.rb | 8 + source/plugins/ruby/in_kube_nodes.rb | 22 +- source/plugins/ruby/in_kube_podinventory.rb | 24 + source/plugins/ruby/in_kube_pvinventory.rb | 7 + .../plugins/ruby/in_kubestate_deployments.rb | 7 + source/plugins/ruby/in_kubestate_hpa.rb | 10 +- source/plugins/ruby/in_win_cadvisor_perf.rb | 14 +- source/plugins/ruby/out_mdm.rb | 13 +- source/plugins/utils/extension.rb | 77 ++++ source/plugins/utils/extension_utils.rb | 27 ++ 29 files changed, 1443 insertions(+), 72 deletions(-) create mode 100644 scripts/dcr-onboarding/ci-extension-dcr-streams.md create mode 100644 scripts/dcr-onboarding/ci-extension-dcr.json create mode 100644 source/plugins/go/src/extension/extension.go create mode 100644 source/plugins/go/src/extension/interfaces.go create mode 100644 source/plugins/go/src/extension/socket_writer.go create mode 100644 source/plugins/go/src/ingestion_token_utils.go create mode 100644 source/plugins/go/src/ingestion_token_utils_test.go create mode 100644 source/plugins/utils/extension.rb create mode 100644 source/plugins/utils/extension_utils.rb diff --git a/build/linux/installer/datafiles/base_container.data b/build/linux/installer/datafiles/base_container.data index de8ccbba0..5d7301aa3 100644 --- a/build/linux/installer/datafiles/base_container.data +++ b/build/linux/installer/datafiles/base_container.data @@ -146,6 +146,9 @@ MAINTAINER: 'Microsoft Corporation' /etc/fluent/plugin/omslog.rb; source/plugins/utils/omslog.rb; 644; root; root /etc/fluent/plugin/oms_common.rb; source/plugins/utils/oms_common.rb; 644; root; root +/etc/fluent/plugin/extension.rb; source/plugins/utils/extension.rb; 644; root; root +/etc/fluent/plugin/extension_utils.rb; source/plugins/utils/extension_utils.rb; 644; root; root + /etc/fluent/kube.conf; build/linux/installer/conf/kube.conf; 644; root; root /etc/fluent/container.conf; build/linux/installer/conf/container.conf; 644; root; root diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index b9e338fa9..7950f30fd 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -58,6 +58,10 @@ else echo "export customResourceId=$AKS_RESOURCE_ID" >> ~/.bashrc source ~/.bashrc echo "customResourceId:$customResourceId" + export customRegion=$AKS_REGION + echo "export customRegion=$AKS_REGION" >> ~/.bashrc + source ~/.bashrc + echo "customRegion:$customRegion" fi #set agent config schema version @@ -430,48 +434,105 @@ DOCKER_CIMPROV_VERSION=$(dpkg -l | grep docker-cimprov | awk '{print $3}') echo "DOCKER_CIMPROV_VERSION=$DOCKER_CIMPROV_VERSION" export DOCKER_CIMPROV_VERSION=$DOCKER_CIMPROV_VERSION echo "export DOCKER_CIMPROV_VERSION=$DOCKER_CIMPROV_VERSION" >> ~/.bashrc -echo "*** activating oneagent in legacy auth mode ***" -CIWORKSPACE_id="$(cat /etc/omsagent-secret/WSID)" -#use the file path as its secure than env -CIWORKSPACE_keyFile="/etc/omsagent-secret/KEY" -cat /etc/mdsd.d/envmdsd | while read line; do - echo $line >> ~/.bashrc -done -source /etc/mdsd.d/envmdsd -echo "setting mdsd workspaceid & key for workspace:$CIWORKSPACE_id" -export CIWORKSPACE_id=$CIWORKSPACE_id -echo "export CIWORKSPACE_id=$CIWORKSPACE_id" >> ~/.bashrc -export CIWORKSPACE_keyFile=$CIWORKSPACE_keyFile -echo "export CIWORKSPACE_keyFile=$CIWORKSPACE_keyFile" >> ~/.bashrc -export OMS_TLD=$domain -echo "export OMS_TLD=$OMS_TLD" >> ~/.bashrc -export MDSD_FLUENT_SOCKET_PORT="29230" -echo "export MDSD_FLUENT_SOCKET_PORT=$MDSD_FLUENT_SOCKET_PORT" >> ~/.bashrc - -#skip imds lookup since not used in legacy auth path -export SKIP_IMDS_LOOKUP_FOR_LEGACY_AUTH="true" -echo "export SKIP_IMDS_LOOKUP_FOR_LEGACY_AUTH=$SKIP_IMDS_LOOKUP_FOR_LEGACY_AUTH" >> ~/.bashrc - -source ~/.bashrc -dpkg -l | grep mdsd | awk '{print $2 " " $3}' +# check if its AAD Auth MSI mode via USING_AAD_MSI_AUTH +export AAD_MSI_AUTH_MODE=false +if [ "${USING_AAD_MSI_AUTH}" == "true" ]; then + echo "*** activating oneagent in aad auth msi mode ***" + export AAD_MSI_AUTH_MODE=true + echo "export AAD_MSI_AUTH_MODE=true" >> ~/.bashrc + + cat /etc/mdsd.d/envmdsd | while read line; do + echo $line >> ~/.bashrc + done + source /etc/mdsd.d/envmdsd + + + export MDSD_FLUENT_SOCKET_PORT="28230" + echo "export MDSD_FLUENT_SOCKET_PORT=$MDSD_FLUENT_SOCKET_PORT" >> ~/.bashrc + export MCS_ENDPOINT="handler.control.monitor.azure.com" + echo "export MCS_ENDPOINT=$MCS_ENDPOINT" >> ~/.bashrc + export AZURE_ENDPOINT="https://monitor.azure.com/" + echo "export AZURE_ENDPOINT=$AZURE_ENDPOINT" >> ~/.bashrc + export ADD_REGION_TO_MCS_ENDPOINT="true" + echo "export ADD_REGION_TO_MCS_ENDPOINT=$ADD_REGION_TO_MCS_ENDPOINT" >> ~/.bashrc + export ENABLE_MCS="true" + echo "export ENABLE_MCS=$ENABLE_MCS" >> ~/.bashrc + export MONITORING_USE_GENEVA_CONFIG_SERVICE="false" + echo "export MONITORING_USE_GENEVA_CONFIG_SERVICE=$MONITORING_USE_GENEVA_CONFIG_SERVICE" >> ~/.bashrc + export MDSD_USE_LOCAL_PERSISTENCY="false" + echo "export MDSD_USE_LOCAL_PERSISTENCY=$MDSD_USE_LOCAL_PERSISTENCY" >> ~/.bashrc + #skip imds metadata lookup since we dont use + #legacy auth misnomer here + export SKIP_IMDS_LOOKUP_FOR_LEGACY_AUTH="true" + echo "export SKIP_IMDS_LOOKUP_FOR_LEGACY_AUTH=$SKIP_IMDS_LOOKUP_FOR_LEGACY_AUTH" >> ~/.bashrc -if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then - echo "starting mdsd with mdsd-port=26130, fluentport=26230 and influxport=26330 in legacy auth mode in sidecar container..." - #use tenant name to avoid unix socket conflict and different ports for port conflict - #roleprefix to use container specific mdsd socket - export TENANT_NAME="${CONTAINER_TYPE}" - echo "export TENANT_NAME=$TENANT_NAME" >> ~/.bashrc - export MDSD_ROLE_PREFIX=/var/run/mdsd-${CONTAINER_TYPE}/default - echo "export MDSD_ROLE_PREFIX=$MDSD_ROLE_PREFIX" >> ~/.bashrc source ~/.bashrc - mkdir /var/run/mdsd-${CONTAINER_TYPE} - # add -T 0xFFFF for full traces - mdsd -r ${MDSD_ROLE_PREFIX} -p 26130 -f 26230 -i 26330 -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos & -else - echo "starting mdsd in legacy auth mode in main container..." - # add -T 0xFFFF for full traces - mdsd -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos & + + dpkg -l | grep mdsd | awk '{print $2 " " $3}' + + if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then + echo "starting mdsd with mdsd-port=26130, fluentport=26230 and influxport=26330 in aad auth msi mode in sidecar container..." + #use tenant name to avoid unix socket conflict and different ports for port conflict + #roleprefix to use container specific mdsd socket + export TENANT_NAME="${CONTAINER_TYPE}" + echo "export TENANT_NAME=$TENANT_NAME" >> ~/.bashrc + export MDSD_ROLE_PREFIX=/var/run/mdsd-${CONTAINER_TYPE}/default + echo "export MDSD_ROLE_PREFIX=$MDSD_ROLE_PREFIX" >> ~/.bashrc + source ~/.bashrc + mkdir /var/run/mdsd-${CONTAINER_TYPE} + # add -T 0xFFFF for full traces + mdsd -a -A -r ${MDSD_ROLE_PREFIX} -p 26130 -f 26230 -i 26330 -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos & + else + echo "starting mdsd in aad auth msi mode in main container..." + # add -T 0xFFFF for full traces + mdsd -a -A -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos & + fi + +else + echo "*** activating oneagent in legacy auth mode ***" + CIWORKSPACE_id="$(cat /etc/omsagent-secret/WSID)" + #use the file path as its secure than env + CIWORKSPACE_keyFile="/etc/omsagent-secret/KEY" + cat /etc/mdsd.d/envmdsd | while read line; do + echo $line >> ~/.bashrc + done + source /etc/mdsd.d/envmdsd + echo "setting mdsd workspaceid & key for workspace:$CIWORKSPACE_id" + export CIWORKSPACE_id=$CIWORKSPACE_id + echo "export CIWORKSPACE_id=$CIWORKSPACE_id" >> ~/.bashrc + export CIWORKSPACE_keyFile=$CIWORKSPACE_keyFile + echo "export CIWORKSPACE_keyFile=$CIWORKSPACE_keyFile" >> ~/.bashrc + export OMS_TLD=$domain + echo "export OMS_TLD=$OMS_TLD" >> ~/.bashrc + export MDSD_FLUENT_SOCKET_PORT="29230" + echo "export MDSD_FLUENT_SOCKET_PORT=$MDSD_FLUENT_SOCKET_PORT" >> ~/.bashrc + + #skip imds lookup since not used in legacy auth path + export SKIP_IMDS_LOOKUP_FOR_LEGACY_AUTH="true" + echo "export SKIP_IMDS_LOOKUP_FOR_LEGACY_AUTH=$SKIP_IMDS_LOOKUP_FOR_LEGACY_AUTH" >> ~/.bashrc + + source ~/.bashrc + + dpkg -l | grep mdsd | awk '{print $2 " " $3}' + + if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then + echo "starting mdsd with mdsd-port=26130, fluentport=26230 and influxport=26330 in legacy auth mode in sidecar container..." + #use tenant name to avoid unix socket conflict and different ports for port conflict + #roleprefix to use container specific mdsd socket + export TENANT_NAME="${CONTAINER_TYPE}" + echo "export TENANT_NAME=$TENANT_NAME" >> ~/.bashrc + export MDSD_ROLE_PREFIX=/var/run/mdsd-${CONTAINER_TYPE}/default + echo "export MDSD_ROLE_PREFIX=$MDSD_ROLE_PREFIX" >> ~/.bashrc + source ~/.bashrc + mkdir /var/run/mdsd-${CONTAINER_TYPE} + # add -T 0xFFFF for full traces + mdsd -r ${MDSD_ROLE_PREFIX} -p 26130 -f 26230 -i 26330 -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos & + else + echo "starting mdsd in legacy auth mode in main container..." + # add -T 0xFFFF for full traces + mdsd -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos & + fi fi # no dependency on fluentd for prometheus side car container diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 17cfb3f77..0276ed7af 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -10,7 +10,7 @@ sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ update-locale LANG=en_US.UTF-8 #install oneagent - Official bits (05/17/2021) -wget https://github.com/microsoft/Docker-Provider/releases/download/05172021-oneagent/azure-mdsd_1.10.1-build.master.213_x86_64.deb +wget https://github.com/microsoft/Docker-Provider/raw/gangams/ci-aad-auth-msi/oneagent-dev/azure-mdsd_1.10.1-build.dev_x86_64.deb /usr/bin/dpkg -i $TMPDIR/azure-mdsd*.deb cp -f $TMPDIR/mdsd.xml /etc/mdsd.d diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index 617c81f38..0a8b39426 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -811,6 +811,9 @@ spec: - mountPath: C:\etc\config\adx name: omsagent-adx-secret readOnly: true + - mountPath: C:\etc\IMDS-access-token + name: imds-token + readOnly: true # Need to mount this only for airgapped clouds - Commenting this since it wont exist in non airgapped clouds # - mountPath: C:\ca # name: ca-certs @@ -871,6 +874,10 @@ spec: secret: secretName: omsagent-adx-secret optional: true + - name: imds-token + secret: + secretName: omsagent-aad-msi-token + optional: true --- kind: Service apiVersion: v1 diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index bc053b0d6..c28554da9 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -43,6 +43,7 @@ function Start-FileSystemWatcher { function Set-EnvironmentVariables { $domain = "opinsights.azure.com" + $mcs_endpoint = "monitor.azure.com" $cloud_environment = "public" if (Test-Path /etc/omsagent-secret/DOMAIN) { # TODO: Change to omsagent-secret before merging @@ -53,11 +54,24 @@ function Set-EnvironmentVariables { # Set DOMAIN [System.Environment]::SetEnvironmentVariable("DOMAIN", $domain, "Process") [System.Environment]::SetEnvironmentVariable("DOMAIN", $domain, "Machine") + + # Set MCS Endpoint + [System.Environment]::SetEnvironmentVariable("MCS_ENDPOINT", $mcs_endpoint, "Process") + [System.Environment]::SetEnvironmentVariable("MCS_ENDPOINT", $mcs_endpoint, "Machine") # Set CLOUD_ENVIRONMENT [System.Environment]::SetEnvironmentVariable("CLOUD_ENVIRONMENT", $cloud_environment, "Process") [System.Environment]::SetEnvironmentVariable("CLOUD_ENVIRONMENT", $cloud_environment, "Machine") + + # Set Region + [System.Environment]::SetEnvironmentVariable("customRegion", $env:AKS_REGION, "Process") + [System.Environment]::SetEnvironmentVariable("customRegion", $env:AKS_REGION, "Machine") + + # Set resource ID + [System.Environment]::SetEnvironmentVariable("customResourceId", $env:AKS_RESOURCE_ID, "Process") + [System.Environment]::SetEnvironmentVariable("customResourceId", $env:AKS_RESOURCE_ID, "Machine") + $wsID = "" if (Test-Path /etc/omsagent-secret/WSID) { # TODO: Change to omsagent-secret before merging @@ -228,6 +242,16 @@ function Set-EnvironmentVariables { else { Write-Host "Failed to set environment variable HOSTNAME for target 'machine' since it is either null or empty" } + # check if its AAD Auth MSI mode via USING_AAD_MSI_AUTH environment variable + $isAADMSIAuth = [System.Environment]::GetEnvironmentVariable("USING_AAD_MSI_AUTH", "process") + if (![string]::IsNullOrEmpty($isAADMSIAuth) -and ($isAADMSIAuth -eq "true")) { + [System.Environment]::SetEnvironmentVariable("AAD_MSI_AUTH_MODE", "true", "Process") + [System.Environment]::SetEnvironmentVariable("AAD_MSI_AUTH_MODE", "true", "Machine") + Write-Host "Using AAD MSI auth" + } + else { + Write-Host "Using LA Legacy Auth" + } # run config parser ruby /opt/omsagentwindows/scripts/ruby/tomlparser.rb diff --git a/scripts/dcr-onboarding/ci-extension-dcr-streams.md b/scripts/dcr-onboarding/ci-extension-dcr-streams.md new file mode 100644 index 000000000..cbac41838 --- /dev/null +++ b/scripts/dcr-onboarding/ci-extension-dcr-streams.md @@ -0,0 +1,186 @@ +# 1 - ContainerLogV2 +> Note- Please note, this table uses NG schema +``` +stream-id: Microsoft-ContainerLogV2 +data-type: CONTAINERINSIGHTS_CONTAINERLOGV2 +intelligence-pack: ContainerInsights +solutions: ContainerInsights +platform: Any +la-table-name: ContainerLogV2 +alias-stream-id: Microsoft-ContainerLogV2 +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` + +# 2 - InsightsMetrics +``` +stream-id: Microsoft-InsightsMetrics +data-type: INSIGHTS_METRICS_BLOB +intelligence-pack: ContainerInsights +solutions: ContainerInsights +platform: Any +la-table-name: InsightsMetrics +alias-stream-id: Microsoft-InsightsMetrics +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` + +# 3 - ContainerInventory + +``` +stream-id: Microsoft-ContainerInventory +data-type: CONTAINER_INVENTORY_BLOB +intelligence-pack: ContainerInsights +solutions: ContainerInsights +platform: Any +la-table-name: ContainerInventory +alias-stream-id: Microsoft-ContainerInventory +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` + +# 4 - ContainerLog + +``` +stream-id: Microsoft-ContainerLog +data-type: CONTAINER_LOG_BLOB +intelligence-pack: Containers +solutions: ContainerInsights +platform: Any +la-table-name: ContainerLog +alias-stream-id: Microsoft-ContainerLog +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` + +# 5 - ContainerNodeInventory + +``` +stream-id: Microsoft-ContainerNodeInventory +data-type: CONTAINER_NODE_INVENTORY_BLOB +intelligence-pack: ContainerInsights +solutions: ContainerInsights +platform: Any +la-table-name: ContainerNodeInventory +alias-stream-id: Microsoft-ContainerNodeInventory +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` + +# 6 - KubePodInventory +``` +stream-id: Microsoft-KubePodInventory +data-type: KUBE_POD_INVENTORY_BLOB +intelligence-pack: ContainerInsights +solutions: ContainerInsights +platform: Any +la-table-name: KubePodInventory +alias-stream-id: Microsoft-KubePodInventory +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` + +# 7 - KubeNodeInventory +``` +stream-id: Microsoft-KubeNodeInventory +data-type: KUBE_NODE_INVENTORY_BLOB +intelligence-pack: ContainerInsights +solutions: ContainerInsights +platform: Any +la-table-name: KubeNodeInventory +alias-stream-id: Microsoft-KubeNodeInventory +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` + +# 8 - KubePVInventory +``` +stream-id: Microsoft-KubePVInventory +data-type: KUBE_PV_INVENTORY_BLOB +intelligence-pack: ContainerInsights +solutions: ContainerInsights +platform: Any +la-table-name: KubePVInventory +alias-stream-id: Microsoft-KubePVInventory +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` + +# 9 - KubeEvents +``` +stream-id: Microsoft-KubeEvents +data-type: KUBE_EVENTS_BLOB +intelligence-pack: ContainerInsights +solutions: ContainerInsights +platform: Any +la-table-name: KubeEvents +alias-stream-id: Microsoft-KubeEvents +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` + +# 10 - KubeServices +``` +stream-id: Microsoft-KubeServices +data-type: KUBE_SERVICES_BLOB +intelligence-pack: ContainerInsights +solutions: ContainerInsights +platform: Any +la-table-name: KubeServices +alias-stream-id: Microsoft-KubeServices +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` + +# 11 - KubeMonAgentEvents +``` +stream-id: Microsoft-KubeMonAgentEvents +data-type: KUBE_MON_AGENT_EVENTS_BLOB +intelligence-pack: Containers +solutions: ContainerInsights +platform: Any +la-table-name: KubeMonAgentEvents +alias-stream-id: Microsoft-KubeMonAgentEvents +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` + +# 12 - KubeHealth +``` +stream-id: Microsoft-KubeHealth +data-type: KUBE_HEALTH_BLOB +intelligence-pack: ContainerInsights +solutions: ContainerInsights +platform: Any +la-table-name: KubeHealth +alias-stream-id: Microsoft-KubeHealth +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` + +# 13 - Perf +``` +> Note - This stream already exists +stream-id: Microsoft-Perf +data-type: LINUX_PERF_BLOB +intelligence-pack: LogManagement +solutions: ContainerInsights +platform: Any +la-table-name: LogManagement +alias-stream-id: Microsoft-Perf +contact-alias: OMScontainers@microsoft.com +stage: to review +tags: agent +``` diff --git a/scripts/dcr-onboarding/ci-extension-dcr.json b/scripts/dcr-onboarding/ci-extension-dcr.json new file mode 100644 index 000000000..03eba1400 --- /dev/null +++ b/scripts/dcr-onboarding/ci-extension-dcr.json @@ -0,0 +1,59 @@ +{ + "location": "", + "properties": { + "dataSources": { + "extensions": [ + { + "name": "ContainerInsightsExtension", + "streams": [ + "Microsoft-Perf", + "Microsoft-ContainerInventory", + "Microsoft-ContainerLog", + "Microsoft-ContainerLogV2", + "Microsoft-ContainerNodeInventory", + "Microsoft-KubeEvents", + "Microsoft-KubeHealth", + "Microsoft-KubeMonAgentEvents", + "Microsoft-KubeNodeInventory", + "Microsoft-KubePodInventory", + "Microsoft-KubePVInventory", + "Microsoft-KubeServices", + "Microsoft-InsightsMetrics" + + ], + "extensionName": "ContainerInsights" + } + ] + }, + "destinations": { + "logAnalytics": [ + { + "workspaceResourceId": "/subscriptions//resourcegroups//providers/microsoft.operationalinsights/workspaces/", + "name": "ciworkspace" + } + ] + }, + "dataFlows": [ + { + "streams": [ + "Microsoft-Perf", + "Microsoft-ContainerInventory", + "Microsoft-ContainerLog", + "Microsoft-ContainerLogV2", + "Microsoft-ContainerNodeInventory", + "Microsoft-KubeEvents", + "Microsoft-KubeHealth", + "Microsoft-KubeMonAgentEvents", + "Microsoft-KubeNodeInventory", + "Microsoft-KubePodInventory", + "Microsoft-KubePVInventory", + "Microsoft-KubeServices", + "Microsoft-InsightsMetrics" + ], + "destinations": [ + "ciworkspace" + ] + } + ] + } +} \ No newline at end of file diff --git a/source/plugins/go/src/extension/extension.go b/source/plugins/go/src/extension/extension.go new file mode 100644 index 000000000..c68140ded --- /dev/null +++ b/source/plugins/go/src/extension/extension.go @@ -0,0 +1,101 @@ +package extension + +import ( + "encoding/json" + "fmt" + "log" + "sync" + "strings" + uuid "github.com/google/uuid" + "github.com/ugorji/go/codec" +) + +type Extension struct { + datatypeStreamIdMap map[string]string +} + +var singleton *Extension +var once sync.Once +var extensionconfiglock sync.Mutex +var logger *log.Logger +var containerType string + +func GetInstance(flbLogger *log.Logger, containerType string) *Extension { + once.Do(func() { + singleton = &Extension{make(map[string]string)} + flbLogger.Println("Extension Instance created") + }) + logger = flbLogger + containerType = containerType + return singleton +} + +func (e *Extension) GetOutputStreamId(datatype string) string { + extensionconfiglock.Lock() + defer extensionconfiglock.Unlock() + if len(e.datatypeStreamIdMap) > 0 && e.datatypeStreamIdMap[datatype] != "" { + message := fmt.Sprintf("OutputstreamId: %s for the datatype: %s", e.datatypeStreamIdMap[datatype], datatype) + logger.Printf(message) + return e.datatypeStreamIdMap[datatype] + } + var err error + e.datatypeStreamIdMap, err = getDataTypeToStreamIdMapping() + if err != nil { + message := fmt.Sprintf("Error getting datatype to streamid mapping: %s", err.Error()) + logger.Printf(message) + } + return e.datatypeStreamIdMap[datatype] +} + +func getDataTypeToStreamIdMapping() (map[string]string, error) { + logger.Printf("extensionconfig::getDataTypeToStreamIdMapping:: getting extension config from fluent socket - start") + guid := uuid.New() + datatypeOutputStreamMap := make(map[string]string) + + taggedData := map[string]interface{}{"Request": "AgentTaggedData", "RequestId": guid.String(), "Tag": "ContainerInsights", "Version": "1"} + jsonBytes, err := json.Marshal(taggedData) + + var data []byte + enc := codec.NewEncoderBytes(&data, new(codec.MsgpackHandle)) + if err := enc.Encode(string(jsonBytes)); err != nil { + return datatypeOutputStreamMap, err + } + + fs := &FluentSocketWriter{ } + fs.sockAddress = "/var/run/mdsd/default_fluent.socket" + if containerType != "" && strings.Compare(strings.ToLower(containerType), "prometheussidecar") == 0 { + fs.sockAddress = fmt.Sprintf("/var/run/mdsd-%s/default_fluent.socket", containerType) + } + responseBytes, err := fs.WriteAndRead(data) + defer fs.disConnect() + logger.Printf("Info::mdsd::Making call to FluentSocket: %s to write and read the config data", fs.sockAddress) + if err != nil { + return datatypeOutputStreamMap, err + } + response := string(responseBytes) + + var responseObjet AgentTaggedDataResponse + err = json.Unmarshal([]byte(response), &responseObjet) + if err != nil { + logger.Printf("Error::mdsd::Failed to unmarshal config data. Error message: %s", string(err.Error())) + return datatypeOutputStreamMap, err + } + + var extensionData TaggedData + json.Unmarshal([]byte(responseObjet.TaggedData), &extensionData) + + extensionConfigs := extensionData.ExtensionConfigs + logger.Printf("Info::mdsd::build the datatype and streamid map -- start") + for _, extensionConfig := range extensionConfigs { + outputStreams := extensionConfig.OutputStreams + for dataType, outputStreamID := range outputStreams { + logger.Printf("Info::mdsd::datatype: %s, outputstreamId: %s", dataType, outputStreamID) + datatypeOutputStreamMap[dataType] = outputStreamID.(string) + } + } + logger.Printf("Info::mdsd::build the datatype and streamid map -- end") + + logger.Printf("extensionconfig::getDataTypeToStreamIdMapping:: getting extension config from fluent socket-end") + + return datatypeOutputStreamMap, nil +} diff --git a/source/plugins/go/src/extension/interfaces.go b/source/plugins/go/src/extension/interfaces.go new file mode 100644 index 000000000..c70ef17b8 --- /dev/null +++ b/source/plugins/go/src/extension/interfaces.go @@ -0,0 +1,34 @@ +package extension + +// AgentTaggedDataResponse struct for response from AgentTaggedData request +type AgentTaggedDataResponse struct { + Request string `json:"Request"` + RequestID string `json:"RequestId"` + Version string `json:"Version"` + Success bool `json:"Success"` + Description string `json:"Description"` + TaggedData string `json:"TaggedData"` +} + +// TaggedData structure for respone +type TaggedData struct { + SchemaVersion int `json:"schemaVersion"` + Version int `json:"version"` + ExtensionName string `json:"extensionName"` + ExtensionConfigs []ExtensionConfig `json:"extensionConfigurations"` + OutputStreamDefinitions map[string]StreamDefinition `json:"outputStreamDefinitions"` +} + +// StreamDefinition structure for named pipes +type StreamDefinition struct { + NamedPipe string `json:"namedPipe"` +} + +// ExtensionConfig structure for extension definition in DCR +type ExtensionConfig struct { + ID string `json:"id"` + OriginIds []string `json:"originIds"` + ExtensionSettings map[string]interface{} `json:"extensionSettings"` + InputStreams map[string]interface{} `json:"inputStreams"` + OutputStreams map[string]interface{} `json:"outputStreams"` +} diff --git a/source/plugins/go/src/extension/socket_writer.go b/source/plugins/go/src/extension/socket_writer.go new file mode 100644 index 000000000..1b16b319c --- /dev/null +++ b/source/plugins/go/src/extension/socket_writer.go @@ -0,0 +1,85 @@ +package extension + +import ( + "net" +) + +//MaxRetries for trying to write data to the socket +const MaxRetries = 5 + +//ReadBufferSize for reading data from sockets +//Current CI extension config size is ~5KB and going with 20KB to handle any future scenarios +const ReadBufferSize = 20480 + +//FluentSocketWriter writes data to AMA's default fluent socket +type FluentSocketWriter struct { + socket net.Conn + sockAddress string +} + +func (fs *FluentSocketWriter) connect() error { + c, err := net.Dial("unix", fs.sockAddress) + if err != nil { + return err + } + fs.socket = c + return nil +} + +func (fs *FluentSocketWriter) disConnect() error { + if (fs.socket != nil) { + fs.socket.Close() + fs.socket = nil + } + return nil +} + +func (fs *FluentSocketWriter) writeWithRetries(data []byte) (int, error) { + var ( + err error + n int + ) + for i := 0; i < MaxRetries; i++ { + n, err = fs.socket.Write(data) + if err == nil { + return n, nil + } + } + if err, ok := err.(net.Error); !ok || !err.Temporary() { + // so that connect() is called next time if write fails + // this happens when mdsd is restarted + _ = fs.socket.Close() // no need to log the socket closing error + fs.socket = nil + } + return 0, err +} + +func (fs *FluentSocketWriter) read() ([]byte, error) { + buf := make([]byte, ReadBufferSize) + n, err := fs.socket.Read(buf) + if err != nil { + return nil, err + } + return buf[:n], nil + +} + +func (fs *FluentSocketWriter) Write(payload []byte) (int, error) { + if fs.socket == nil { + // previous write failed with permanent error and socket was closed. + if err := fs.connect(); err != nil { + return 0, err + } + } + + return fs.writeWithRetries(payload) +} + +//WriteAndRead writes data to the socket and sends the response back +func (fs *FluentSocketWriter) WriteAndRead(payload []byte) ([]byte, error) { + _, err := fs.Write(payload) + if err != nil { + return nil, err + } + return fs.read() +} diff --git a/source/plugins/go/src/ingestion_token_utils.go b/source/plugins/go/src/ingestion_token_utils.go new file mode 100644 index 000000000..4dffd7974 --- /dev/null +++ b/source/plugins/go/src/ingestion_token_utils.go @@ -0,0 +1,429 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "time" +) + +const IMDSTokenPathForWindows = "c:/etc/imds-access-token/token" // only used in windows + +var IMDSToken string +var IMDSTokenExpiration int64 + +var ConfigurationId string +var ChannelId string + +var IngestionAuthToken string +var IngestionAuthTokenExpiration int64 + +type IMDSResponse struct { + AccessToken string `json:"access_token"` + ClientID string `json:"client_id"` + ExpiresIn string `json:"expires_in"` + ExpiresOn string `json:"expires_on"` + ExtExpiresIn string `json:"ext_expires_in"` + NotBefore string `json:"not_before"` + Resource string `json:"resource"` + TokenType string `json:"token_type"` +} + +type AgentConfiguration struct { + Configurations []struct { + Configurationid string `json:"configurationId"` + Etag string `json:"eTag"` + Op string `json:"op"` + Content struct { + Datasources []struct { + Configuration struct { + Extensionname string `json:"extensionName"` + } `json:"configuration"` + ID string `json:"id"` + Kind string `json:"kind"` + Streams []struct { + Stream string `json:"stream"` + Solution string `json:"solution"` + Extensionoutputstream string `json:"extensionOutputStream"` + } `json:"streams"` + Sendtochannels []string `json:"sendToChannels"` + } `json:"dataSources"` + Channels []struct { + Endpoint string `json:"endpoint"` + ID string `json:"id"` + Protocol string `json:"protocol"` + } `json:"channels"` + Extensionconfigurations struct { + Containerinsights []struct { + ID string `json:"id"` + Originids []string `json:"originIds"` + //TODO: make this a map so that if more types are added in the future it won't break. Also test if removing a type breaks the json deserialization + Outputstreams struct { + LinuxPerfBlob string `json:"LINUX_PERF_BLOB"` + ContainerInventoryBlob string `json:"CONTAINER_INVENTORY_BLOB"` + ContainerLogBlob string `json:"CONTAINER_LOG_BLOB"` + ContainerinsightsContainerlogv2 string `json:"CONTAINERINSIGHTS_CONTAINERLOGV2"` + ContainerNodeInventoryBlob string `json:"CONTAINER_NODE_INVENTORY_BLOB"` + KubeEventsBlob string `json:"KUBE_EVENTS_BLOB"` + KubeHealthBlob string `json:"KUBE_HEALTH_BLOB"` + KubeMonAgentEventsBlob string `json:"KUBE_MON_AGENT_EVENTS_BLOB"` + KubeNodeInventoryBlob string `json:"KUBE_NODE_INVENTORY_BLOB"` + KubePodInventoryBlob string `json:"KUBE_POD_INVENTORY_BLOB"` + KubePvInventoryBlob string `json:"KUBE_PV_INVENTORY_BLOB"` + KubeServicesBlob string `json:"KUBE_SERVICES_BLOB"` + InsightsMetricsBlob string `json:"INSIGHTS_METRICS_BLOB"` + } `json:"outputStreams"` + } `json:"ContainerInsights"` + } `json:"extensionConfigurations"` + } `json:"content"` + } `json:"configurations"` +} + +type IngestionTokenResponse struct { + Configurationid string `json:"configurationId"` + Ingestionauthtoken string `json:"ingestionAuthToken"` +} + +func getIngestionToken() (authToken string, err error) { + // check if the ingestion token has not been fetched yet or is expiring soon. If so then re-fetch it first. + // TODO: re-fetch token in a new thread if it is about to expire (don't block the main thread) + + if IMDSToken == "" || IMDSTokenExpiration <= time.Now().Unix()+30 { // refresh the token 30 seconds before it expires + IMDSToken, IMDSTokenExpiration, err = getAccessTokenFromIMDS(true) //TODO: read from an env variable? Or maybe from OS? + if err != nil { + message := fmt.Sprintf("Error on getAccessTokenFromIMDS %s \n", err.Error()) + Log(message) + SendException(message) + return "", err + } + } + + // ignore agent configuration expiring, the configuration and channel IDs will never change (without creating an agent restart) + if ConfigurationId == "" || ChannelId == "" { + ConfigurationId, ChannelId, _, err = getAgentConfiguration(IMDSToken) + if err != nil { + message := fmt.Sprintf("Error getAgentConfiguration %s \n", err.Error()) + Log(message) + SendException(message) + return "", err + } + } + + if IngestionAuthToken == "" || IngestionAuthTokenExpiration <= time.Now().Unix()+10 { + IngestionAuthToken, IngestionAuthTokenExpiration, err = getIngestionAuthToken(IMDSToken, ConfigurationId, ChannelId) + if err != nil { + message := fmt.Sprintf("Error getIngestionAuthToken %s \n", err.Error()) + Log(message) + SendException(message) + return "", err + } + } + + return IngestionAuthToken, nil +} + +func getAccessTokenFromIMDS(fromfile bool) (string, int64, error) { + Log("Info getAccessTokenFromIMDS: start") + MCS_ENDPOINT := os.Getenv("MCS_ENDPOINT") + imdsAccessToken := "" + + var responseBytes []byte + var err error + + if fromfile { + // only used in windows + responseBytes, err = ioutil.ReadFile(IMDSTokenPathForWindows) + if err != nil { + Log("getAccessTokenFromIMDS: Could not read IMDS token from file", err) + return imdsAccessToken, 0, err + } + } else { + var msi_endpoint *url.URL + msi_endpoint, err := url.Parse("http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://" + MCS_ENDPOINT + "/") + if err != nil { + Log("getAccessTokenFromIMDS: Error creating IMDS endpoint URL: %s", err.Error()) + return imdsAccessToken, 0, err + } + req, err := http.NewRequest("GET", msi_endpoint.String(), nil) + if err != nil { + Log("getAccessTokenFromIMDS: Error creating HTTP request: %s", err.Error()) + return imdsAccessToken, 0, err + } + req.Header.Add("Metadata", "true") + + // Call managed services for Azure resources token endpoint + var resp *http.Response = nil + for i := 0; i < 3; i++ { + // client := &http.Client{} + resp, err = HTTPClient.Do(req) + if err != nil { + message := fmt.Sprintf("getAccessTokenFromIMDS: Error calling token endpoint: %s", err.Error()) + Log(message) + SendException(message) // send the exception here because this error is not returned. The calling function will send any returned errors to telemetry. + continue + } + //TODO: is this the best place to defer closing the response body? + defer resp.Body.Close() + + Log("getAccessTokenFromIMDS: IMDS Response Status: %d", resp.StatusCode) + if resp.StatusCode != 200 { + message := fmt.Sprintf("getAccessTokenFromIMDS: IMDS Request failed with an error code : %d", resp.StatusCode) + Log(message) + SendException(message) + continue + } + break // call succeeded, don't retry any more + } + if resp == nil { + Log("getAccessTokenFromIMDS: IMDS Request ran out of retries") + return imdsAccessToken, 0, err + } + + // Pull out response body + responseBytes, err = ioutil.ReadAll(resp.Body) + if err != nil { + Log("getAccessTokenFromIMDS: Error reading response body : %s", err.Error()) + return imdsAccessToken, 0, err + } + } + + // Unmarshall response body into struct + var imdsResponse IMDSResponse + err = json.Unmarshal(responseBytes, &imdsResponse) + if err != nil { + Log("getAccessTokenFromIMDS: Error unmarshalling the response: %s", err.Error()) + return imdsAccessToken, 0, err + } + imdsAccessToken = imdsResponse.AccessToken + + expiration, err := strconv.ParseInt(imdsResponse.ExpiresOn, 10, 64) + if err != nil { + Log("Error reading expiration time from response header body: %s", err.Error()) + return imdsAccessToken, 0, err + } + + Log("IMDS Token obtained: %s", imdsAccessToken) + + Log("Info getAccessTokenFromIMDS: end") + + return imdsAccessToken, expiration, nil +} + +func getAgentConfiguration(imdsAccessToken string) (configurationId string, channelId string, expiration int64, err error) { + Log("Info getAgentConfiguration: start") + configurationId = "" + channelId = "" + apiVersion := "2020-08-01-preview" + var amcs_endpoint *url.URL + resourceId := os.Getenv("customResourceId") + resourceRegion := os.Getenv("customRegion") + MCS_ENDPOINT := os.Getenv("MCS_ENDPOINT") + amcs_endpoint_string := fmt.Sprintf("https://%s.handler.control."+MCS_ENDPOINT+"%s/agentConfigurations?platform=windows&api-version=%s", resourceRegion, resourceId, apiVersion) + amcs_endpoint, err = url.Parse(amcs_endpoint_string) + + var bearer = "Bearer " + imdsAccessToken + // Create a new request using http + req, err := http.NewRequest("GET", amcs_endpoint.String(), nil) + if err != nil { + message := fmt.Sprintf("Error creating HTTP request for AMCS endpoint: %s", err.Error()) + Log(message) + return configurationId, channelId, 0, err + } + + // add authorization header to the req + req.Header.Add("Authorization", bearer) + + var resp *http.Response = nil + + for i := 0; i < 3; i++ { + // Call managed services for Azure resources token endpoint + // client := &http.Client{} + resp, err = HTTPClient.Do(req) + if err != nil { + message := fmt.Sprintf("Error calling amcs endpoint: %s", err.Error()) + Log(message) + SendException(message) + continue + } + + //TODO: is this the best place to defer closing the response body? + defer resp.Body.Close() + + Log("getAgentConfiguration Response Status: %d", resp.StatusCode) + if resp.StatusCode != 200 { + message := fmt.Sprintf("getAgentConfiguration Request failed with an error code : %d", resp.StatusCode) + Log(message) + SendException(message) + continue + } + break // call succeeded, don't retry any more + } + if resp == nil { + Log("getAgentConfiguration Request ran out of retries") + return configurationId, channelId, 0, err + } + + // get content timeout + expiration, err = getExpirationFromAmcsResponse(resp.Header) + if err != nil { + Log("getAgentConfiguration failed to parse auth token timeout") + return configurationId, channelId, 0, err + } + Log(fmt.Sprintf("getAgentConfiguration: DCR expires at %d seconds (unix timestamp)", expiration)) + + // Pull out response body + //TODO: does responseBytes need to be closed? + responseBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + Log("Error reading response body from AMCS API call : %s", err.Error()) + return configurationId, channelId, expiration, err + } + + // Unmarshall response body into struct + var agentConfiguration AgentConfiguration + err = json.Unmarshal(responseBytes, &agentConfiguration) + if err != nil { + Log("Error unmarshalling the response: %s", err.Error()) + return configurationId, channelId, expiration, err + } + + if len(agentConfiguration.Configurations) == 0 { + Log("Received empty agentConfiguration.Configurations array") + return configurationId, channelId, expiration, err + } + + if len(agentConfiguration.Configurations[0].Content.Channels) == 0 { + Log("Received empty agentConfiguration.Configurations[0].Content.Channels") + return configurationId, channelId, expiration, err + } + + configurationId = agentConfiguration.Configurations[0].Configurationid + channelId = agentConfiguration.Configurations[0].Content.Channels[0].ID + + Log("obtained configurationId: %s, channelId: %s", configurationId, channelId) + + Log("Info getAgentConfiguration: end") + + return configurationId, channelId, expiration, nil +} + +func getIngestionAuthToken(imdsAccessToken string, configurationId string, channelId string) (ingestionAuthToken string, expiration int64, err error) { + Log("Info getIngestionAuthToken: start") + ingestionAuthToken = "" + var amcs_endpoint *url.URL + resourceId := os.Getenv("customResourceId") + resourceRegion := os.Getenv("customRegion") + MCS_ENDPOINT := os.Getenv("MCS_ENDPOINT") + apiVersion := "2020-04-01-preview" + amcs_endpoint_string := fmt.Sprintf("https://%s.handler.control."+MCS_ENDPOINT+"%s/agentConfigurations/%s/channels/%s/issueIngestionToken?platform=windows&api-version=%s", resourceRegion, resourceId, configurationId, channelId, apiVersion) + amcs_endpoint, err = url.Parse(amcs_endpoint_string) + + var bearer = "Bearer " + imdsAccessToken + + // Create a new request using http + req, err := http.NewRequest("GET", amcs_endpoint.String(), nil) + if err != nil { + Log("Error creating HTTP request for AMCS endpoint: %s", err.Error()) + return ingestionAuthToken, 0, err + } + + // add authorization header to the req + req.Header.Add("Authorization", bearer) + + var resp *http.Response = nil + + for i := 0; i < 3; i++ { + // Call managed services for Azure resources token endpoint + resp, err = HTTPClient.Do(req) + if err != nil { + message := fmt.Sprintf("Error calling amcs endpoint for ingestion auth token: %s", err.Error()) + Log(message) + SendException(message) + resp = nil + continue + } + + //TODO: is this the best place to defer closing the response body? + defer resp.Body.Close() + + Log("getIngestionAuthToken Response Status: %d", resp.StatusCode) + if resp.StatusCode != 200 { + message := fmt.Sprintf("getIngestionAuthToken Request failed with an error code : %d", resp.StatusCode) + Log(message) + SendException(message) + resp = nil + continue + } + break + } + + if resp == nil { + Log("ran out of retries calling AMCS for ingestion token") + return ingestionAuthToken, 0, err + } + + expiration, err = getExpirationFromAmcsResponse(resp.Header) + if err != nil { + Log("getIngestionAuthToken failed to parse auth token expiration time") + return ingestionAuthToken, 0, err + } + Log("getIngestionAuthToken: token times out at time %d (unix timestamp)", expiration) + + // Pull out response body + responseBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + Log("Error reading response body from AMCS Ingestion API call : %s", err.Error()) + return ingestionAuthToken, expiration, err + } + + // Unmarshall response body into struct + var ingestionTokenResponse IngestionTokenResponse + err = json.Unmarshal(responseBytes, &ingestionTokenResponse) + if err != nil { + Log("Error unmarshalling the response: %s", err.Error()) + return ingestionAuthToken, expiration, err + } + + ingestionAuthToken = ingestionTokenResponse.Ingestionauthtoken + + // Log("ingestionAuthToken obtained: %s", ingestionAuthToken) + + Log("Info getIngestionAuthToken: end") + return ingestionAuthToken, expiration, nil +} + +var cacheControlHeaderRegex = regexp.MustCompile(`max-age=([0-9]+)`) + +func getExpirationFromAmcsResponse(header http.Header) (bestBeforeTime int64, err error) { + // Get a timeout from a AMCS HTTP response header + timeout := 0 + + cacheControlHeader, valueInMap := header["Cache-Control"] + if !valueInMap { + return 0, errors.New("Cache-Control not in passed header") + } + + for _, entry := range cacheControlHeader { + match := cacheControlHeaderRegex.FindStringSubmatch(entry) + if len(match) == 2 { + timeout, err = strconv.Atoi(match[1]) + if err != nil { + Log("getIngestionAuthToken: error getting timeout from auth token. Header: " + strings.Join(cacheControlHeader, ",")) + return 0, err + } + + bestBeforeTime = time.Now().Unix() + int64(timeout) + + return bestBeforeTime, nil + } + } + + return 0, errors.New("didn't find timeout in header") +} diff --git a/source/plugins/go/src/ingestion_token_utils_test.go b/source/plugins/go/src/ingestion_token_utils_test.go new file mode 100644 index 000000000..f987a2408 --- /dev/null +++ b/source/plugins/go/src/ingestion_token_utils_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "net/http" + "testing" + "time" +) + +// TODO: add unit tests for retry code. + +type headerElementTestType = struct { + in http.Header + want int64 + wantError bool +} + +func TestGetTimeoutFromAmcsResponse(t *testing.T) { + + var cases []headerElementTestType + + cases = append(cases, headerElementTestType{http.Header{ + "Request-Context": {"appId=cid-v1:*******-*********-*********"}, + "Api-Supported-Versions": {"2019-11-02-preview", "2020-04-01-preview", "2020-08-01-preview"}, + "Date": {"Thu, 06 May 2021 21:17:42 GMT"}, + "Cache-Control": {"public", "max-age=3600"}, + "Content-Type": {"application/json; charset=utf-8"}, + "Vary": {"Accept-Encoding"}, + "Server": {"Microsoft-HTTPAPI/2.0"}, + }, 3600, false}) + + cases = append(cases, headerElementTestType{http.Header{ + "Request-Context": {"appId=cid-v1:*******-*********-*********"}, + "Api-Supported-Versions": {"2019-11-02-preview", "2020-04-01-preview", "2020-08-01-preview"}, + "Date": {"Thu, 06 May 2021 21:17:42 GMT"}, + "Cache-Control": {"public", "max-age=300"}, + "Content-Type": {"application/json; charset=utf-8"}, + "Vary": {"Accept-Encoding"}, + "Server": {"Microsoft-HTTPAPI/2.0"}, + }, 300, false}) + + cases = append(cases, headerElementTestType{http.Header{ + "Request-Context": {"appId=cid-v1:*******-*********-*********"}, + }, 0, true}) + + for _, c := range cases { + got, err := getExpirationFromAmcsResponse(c.in) + if (err != nil) != c.wantError { + t.Errorf("getTimeoutFromAmcsResponse() did not return correct error, expected: %v, got %s", c.wantError, err) + } else if err == nil && (got < c.want+time.Now().Unix()-1 || got > c.want+time.Now().Unix()) { + // Adding a range of 1 second should be enough to keep this test from being flaky, but it might have to be re-thought. + t.Errorf("getTimeoutFromAmcsResponse() (%v) == %d, want %d", c.in, got, c.want+time.Now().Unix()) + } + } +} diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index 25f364c55..814f411ba 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -22,6 +22,7 @@ import ( "github.com/tinylib/msgp/msgp" lumberjack "gopkg.in/natefinch/lumberjack.v2" + "Docker-Provider/source/plugins/go/src/extension" "github.com/Azure/azure-kusto-go/kusto/ingest" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -106,6 +107,11 @@ const ContainerLogsV1Route = "v1" //container logs schema (v2=ContainerLogsV2 table in LA, anything else ContainerLogs table in LA. This is applicable only if Container logs route is NOT ADX) const ContainerLogV2SchemaVersion = "v2" +//env variable for AAD MSI Auth mode +const AADMSIAuthMode = "AAD_MSI_AUTH_MODE" + +// Tag prefix of mdsd output streamid for AMA in MSI auth mode +const MdsdOutputStreamIdTagPrefix = "dcr-" //env variable to container type const ContainerTypeEnv = "CONTAINER_TYPE" @@ -168,7 +174,9 @@ var ( // flag to check if its Windows OS IsWindows bool // container type - ContainerType string + ContainerType string + // flag to check whether LA AAD MSI Auth Enabled or not + IsAADMSIAuthMode bool ) var ( @@ -702,7 +710,11 @@ func flushKubeMonAgentEventRecords() { } } } - if (IsWindows == false && len(msgPackEntries) > 0) { //for linux, mdsd route + if (IsWindows == false && len(msgPackEntries) > 0) { //for linux, mdsd route + if IsAADMSIAuthMode == true && strings.HasPrefix(MdsdKubeMonAgentEventsTagName, MdsdOutputStreamIdTagPrefix) == false { + Log("Info::mdsd::obtaining output stream id for data type: %s", KubeMonAgentEventDataType) + MdsdKubeMonAgentEventsTagName = extension.GetInstance(FLBLogger, ContainerType).GetOutputStreamId(KubeMonAgentEventDataType) + } Log("Info::mdsd:: using mdsdsource name for KubeMonAgentEvents: %s", MdsdKubeMonAgentEventsTagName) msgpBytes := convertMsgPackEntriesToMsgpBytes(MdsdKubeMonAgentEventsTagName, msgPackEntries) if MdsdKubeMonMsgpUnixSocketClient == nil { @@ -760,6 +772,17 @@ func flushKubeMonAgentEventRecords() { req.Header.Set("x-ms-AzureResourceId", ResourceID) } + if IsAADMSIAuthMode == true { + ingestionAuthToken, err := getIngestionToken() + if err != nil { + Log("failed to get ODS Ingestion Auth Token (even after retries)") + message := fmt.Sprintf("Error string: %s \n", err.Error()) + Log(message) + } + // add authorization header to the req + req.Header.Set("Authorization", "Bearer "+ingestionAuthToken) + } + resp, err := HTTPClient.Do(req) elapsed = time.Since(start) @@ -904,7 +927,11 @@ func PostTelegrafMetricsToLA(telegrafRecords []map[interface{}]interface{}) int } } } - if (len(msgPackEntries) > 0) { + if (len(msgPackEntries) > 0) { + if IsAADMSIAuthMode == true && (strings.HasPrefix(MdsdInsightsMetricsTagName, MdsdOutputStreamIdTagPrefix) == false) { + Log("Info::mdsd::obtaining output stream id for InsightsMetricsDataType since Log Analytics AAD MSI Auth Enabled") + MdsdInsightsMetricsTagName = extension.GetInstance(FLBLogger, ContainerType).GetOutputStreamId(InsightsMetricsDataType) + } msgpBytes := convertMsgPackEntriesToMsgpBytes(MdsdInsightsMetricsTagName, msgPackEntries) if MdsdInsightsMetricsMsgpUnixSocketClient == nil { Log("Error::mdsd::mdsd connection does not exist. re-connecting ...") @@ -979,6 +1006,17 @@ func PostTelegrafMetricsToLA(telegrafRecords []map[interface{}]interface{}) int if ResourceCentric == true { req.Header.Set("x-ms-AzureResourceId", ResourceID) } + if IsAADMSIAuthMode == true { + ingestionAuthToken, err := getIngestionToken() + if err != nil { + Log("failed to get ODS Ingestion Auth Token (even after retries)") + message := fmt.Sprintf("Error string: %s \n", err.Error()) + Log(message) + return output.FLB_RETRY + } + // add authorization header to the req + req.Header.Set("Authorization", "Bearer "+ingestionAuthToken) + } start := time.Now() resp, err := HTTPClient.Do(req) @@ -1184,6 +1222,16 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { if len(msgPackEntries) > 0 && ContainerLogsRouteV2 == true { //flush to mdsd + if IsAADMSIAuthMode == true && strings.HasPrefix(MdsdContainerLogTagName, MdsdOutputStreamIdTagPrefix) == false { + Log("Info::mdsd::obtaining output stream id") + if ContainerLogSchemaV2 == true { + MdsdContainerLogTagName = extension.GetInstance(FLBLogger, ContainerType).GetOutputStreamId(ContainerLogV2DataType) + } else { + MdsdContainerLogTagName = extension.GetInstance(FLBLogger, ContainerType).GetOutputStreamId(ContainerLogDataType) + } + Log("Info::mdsd:: using mdsdsource name: %s", MdsdContainerLogTagName) + } + fluentForward := MsgPackForward{ Tag: MdsdContainerLogTagName, Entries: msgPackEntries, @@ -1343,6 +1391,19 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { req.Header.Set("x-ms-AzureResourceId", ResourceID) } + if IsAADMSIAuthMode == true { + ingestionAuthToken, err := getIngestionToken() + if err != nil { + Log("failed to get ODS Ingestion Auth Token (even after retries)") + message := fmt.Sprintf("Error string: %s \n", err.Error()) + Log(message) + return output.FLB_RETRY + } + + // add authorization header to the req + req.Header.Set("Authorization", "Bearer "+ingestionAuthToken) + } + resp, err := HTTPClient.Do(req) elapsed = time.Since(start) @@ -1439,8 +1500,7 @@ func GetContainerIDK8sNamespacePodNameFromFileName(filename string) (string, str } // InitializePlugin reads and populates plugin configuration -func InitializePlugin(pluginConfPath string, agentVersion string) { - +func InitializePlugin(pluginConfPath string, agentVersion string) { go func() { isTest := os.Getenv("ISTEST") if strings.Compare(strings.ToLower(strings.TrimSpace(isTest)), "true") == 0 { @@ -1541,6 +1601,10 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { } Log("OMSEndpoint %s", OMSEndpoint) + IsAADMSIAuthMode = false + if strings.Compare(strings.ToLower(os.Getenv(AADMSIAuthMode)), "true") == 0 { + Log("AAD MSI Auth Mode Configured") + } ResourceID = os.Getenv(envAKSResourceID) if len(ResourceID) > 0 { diff --git a/source/plugins/go/src/utils.go b/source/plugins/go/src/utils.go index 3fe5c6d0e..02d30607e 100644 --- a/source/plugins/go/src/utils.go +++ b/source/plugins/go/src/utils.go @@ -63,27 +63,32 @@ func ReadConfiguration(filename string) (map[string]string, error) { // CreateHTTPClient used to create the client for sending post requests to OMSEndpoint func CreateHTTPClient() { - certFilePath := PluginConfiguration["cert_file_path"] - keyFilePath := PluginConfiguration["key_file_path"] - if IsWindows == false { - certFilePath = fmt.Sprintf(certFilePath, WorkspaceID) - keyFilePath = fmt.Sprintf(keyFilePath, WorkspaceID) - } - cert, err := tls.LoadX509KeyPair(certFilePath, keyFilePath) - if err != nil { - message := fmt.Sprintf("Error when loading cert %s", err.Error()) - SendException(message) - time.Sleep(30 * time.Second) - Log(message) - log.Fatalf("Error when loading cert %s", err.Error()) - } + var transport *http.Transport + if IsAADMSIAuthMode { + transport = &http.Transport{} + } else { + certFilePath := PluginConfiguration["cert_file_path"] + keyFilePath := PluginConfiguration["key_file_path"] + if IsWindows == false { + certFilePath = fmt.Sprintf(certFilePath, WorkspaceID) + keyFilePath = fmt.Sprintf(keyFilePath, WorkspaceID) + } + cert, err := tls.LoadX509KeyPair(certFilePath, keyFilePath) + if err != nil { + message := fmt.Sprintf("Error when loading cert %s", err.Error()) + SendException(message) + time.Sleep(30 * time.Second) + Log(message) + log.Fatalf("Error when loading cert %s", err.Error()) + } - tlsConfig := &tls.Config{ - Certificates: []tls.Certificate{cert}, - } + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + } - tlsConfig.BuildNameToCertificate() - transport := &http.Transport{TLSClientConfig: tlsConfig} + tlsConfig.BuildNameToCertificate() + transport = &http.Transport{TLSClientConfig: tlsConfig} + } // set the proxy if the proxy configured if ProxyEndpoint != "" { proxyEndpointUrl, err := url.Parse(ProxyEndpoint) diff --git a/source/plugins/ruby/ApplicationInsightsUtility.rb b/source/plugins/ruby/ApplicationInsightsUtility.rb index 31f9503cd..eaa1d903d 100644 --- a/source/plugins/ruby/ApplicationInsightsUtility.rb +++ b/source/plugins/ruby/ApplicationInsightsUtility.rb @@ -21,6 +21,8 @@ class ApplicationInsightsUtility @@EnvApplicationInsightsEndpoint = "APPLICATIONINSIGHTS_ENDPOINT" @@EnvControllerType = "CONTROLLER_TYPE" @@EnvContainerRuntime = "CONTAINER_RUNTIME" + @@EnvAADMSIAuthMode = "AAD_MSI_AUTH_MODE" + @@isWindows = false @@hostName = (OMS::Common.get_hostname) @@os_type = ENV["OS_TYPE"] @@ -82,7 +84,12 @@ def initializeUtility() isProxyConfigured = false $log.info("proxy is not configured") end - + aadAuthMSIMode = ENV[@@EnvAADMSIAuthMode] + if !aadAuthMSIMode.nil? && !aadAuthMSIMode.empty? && aadAuthMSIMode.downcase == "true".downcase + @@CustomProperties["aadAuthMSIMode"] = "true" + else + @@CustomProperties["aadAuthMSIMode"] = "false" + end #Check if telemetry is turned off telemetryOffSwitch = ENV["DISABLE_TELEMETRY"] if telemetryOffSwitch && !telemetryOffSwitch.nil? && !telemetryOffSwitch.empty? && telemetryOffSwitch.downcase == "true".downcase diff --git a/source/plugins/ruby/constants.rb b/source/plugins/ruby/constants.rb index c037c99f6..0971a9b96 100644 --- a/source/plugins/ruby/constants.rb +++ b/source/plugins/ruby/constants.rb @@ -103,5 +103,28 @@ class Constants #Pod Statuses POD_STATUS_TERMINATING = "Terminating" + # Data type ids + CONTAINER_INVENTORY_DATA_TYPE = "CONTAINER_INVENTORY_BLOB" + CONTAINER_NODE_INVENTORY_DATA_TYPE = "CONTAINER_NODE_INVENTORY_BLOB" + PERF_DATA_TYPE = "LINUX_PERF_BLOB" + INSIGHTS_METRICS_DATA_TYPE = "INSIGHTS_METRICS_BLOB" + KUBE_SERVICES_DATA_TYPE = "KUBE_SERVICES_BLOB" + KUBE_POD_INVENTORY_DATA_TYPE = "KUBE_POD_INVENTORY_BLOB" + KUBE_NODE_INVENTORY_DATA_TYPE = "KUBE_NODE_INVENTORY_BLOB" + KUBE_PV_INVENTORY_DATA_TYPE = "KUBE_PV_INVENTORY_BLOB" + KUBE_EVENTS_DATA_TYPE = "KUBE_EVENTS_BLOB" + KUBE_MON_AGENT_EVENTS_DATA_TYPE = "KUBE_MON_AGENT_EVENTS_BLOB" + KUBE_HEALTH_DATA_TYPE = "KUBE_HEALTH_BLOB" + CONTAINERLOGV2_DATA_TYPE = "CONTAINERINSIGHTS_CONTAINERLOGV2" + CONTAINERLOG_DATA_TYPE = "CONTAINER_LOG_BLOB" + + #ContainerInsights Extension (AMCS) + CI_EXTENSION_NAME = "ContainerInsights" + CI_EXTENSION_VERSION = "1" + #Current CI extension config size is ~5KB and going with 20KB to handle any future scenarios + CI_EXTENSION_CONFIG_MAX_BYTES = 20480 + ONEAGENT_FLUENT_SOCKET_NAME = "/var/run/mdsd/default_fluent.socket" + #Tag prefix for output stream + EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX = "dcr-" end diff --git a/source/plugins/ruby/filter_health_model_builder.rb b/source/plugins/ruby/filter_health_model_builder.rb index d491f17c2..bd42260e2 100644 --- a/source/plugins/ruby/filter_health_model_builder.rb +++ b/source/plugins/ruby/filter_health_model_builder.rb @@ -4,7 +4,8 @@ require 'fluent/plugin/filter' -module Fluent::Plugin +module Fluent::Plugin + require_relative 'extension_utils' require 'logger' require 'yajl/json_gem' Dir[File.join(__dir__, './health', '*.rb')].each { |file| require file } @@ -91,6 +92,13 @@ def filter_stream(tag, es) begin new_es = Fluent::MultiEventStream.new time = Time.now + if ExtensionUtils.isAADMSIAuthMode() + $log.info("filter_health_model_builder::enumerate: AAD AUTH MSI MODE") + if !@rewrite_tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @rewrite_tag = ExtensionUtils.getOutputStreamId(Constants::KUBE_HEALTH_DATA_TYPE) + end + $log.info("filter_health_model_builder::filter_stream: using tag -#{@rewrite_tag} @ #{Time.now.utc.iso8601}") + end if tag.start_with?("kubehealth.DaemonSet.Node") node_records = [] diff --git a/source/plugins/ruby/in_cadvisor_perf.rb b/source/plugins/ruby/in_cadvisor_perf.rb index b3f9bd08b..da42e4560 100644 --- a/source/plugins/ruby/in_cadvisor_perf.rb +++ b/source/plugins/ruby/in_cadvisor_perf.rb @@ -20,7 +20,8 @@ def initialize require_relative "CAdvisorMetricsAPIClient" require_relative "oms_common" require_relative "omslog" - require_relative "constants" + require_relative "constants" + require_relative "extension_utils" end config_param :run_interval, :time, :default => 60 @@ -68,6 +69,17 @@ def enumerate() eventStream.add(time, record) if record end + if ExtensionUtils.isAADMSIAuthMode() + $log.info("in_cadvisor_perf::enumerate: AAD AUTH MSI MODE") + if !@tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @tag = ExtensionUtils.getOutputStreamId(Constants::PERF_DATA_TYPE) + end + if !@insightsmetricstag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @insightsmetricstag = ExtensionUtils.getOutputStreamId(Constants::INSIGHTS_METRICS_DATA_TYPE) + end + $log.info("in_cadvisor_perf::enumerate: using perf tag -#{@tag} @ #{Time.now.utc.iso8601}") + $log.info("in_cadvisor_perf::enumerate: using insightsmetrics tag -#{@insightsmetricstag} @ #{Time.now.utc.iso8601}") + end router.emit_stream(@tag, eventStream) if eventStream router.emit_stream(@mdmtag, eventStream) if eventStream router.emit_stream(@containerhealthtag, eventStream) if eventStream diff --git a/source/plugins/ruby/in_containerinventory.rb b/source/plugins/ruby/in_containerinventory.rb index eebf422d6..6de1b0bb8 100644 --- a/source/plugins/ruby/in_containerinventory.rb +++ b/source/plugins/ruby/in_containerinventory.rb @@ -17,7 +17,8 @@ def initialize require_relative "ApplicationInsightsUtility" require_relative "omslog" require_relative "CAdvisorMetricsAPIClient" - require_relative "kubernetes_container_inventory" + require_relative "kubernetes_container_inventory" + require_relative "extension_utils" end config_param :run_interval, :time, :default => 60 @@ -57,6 +58,13 @@ def enumerate eventStream = Fluent::MultiEventStream.new hostName = "" $log.info("in_container_inventory::enumerate : Begin processing @ #{Time.now.utc.iso8601}") + if ExtensionUtils.isAADMSIAuthMode() + $log.info("in_container_inventory::enumerate: AAD AUTH MSI MODE") + if !@tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @tag = ExtensionUtils.getOutputStreamId(Constants::CONTAINER_INVENTORY_DATA_TYPE) + end + $log.info("in_container_inventory::enumerate: using tag -#{@tag} @ #{Time.now.utc.iso8601}") + end begin containerRuntimeEnv = ENV["CONTAINER_RUNTIME"] $log.info("in_container_inventory::enumerate : container runtime : #{containerRuntimeEnv}") diff --git a/source/plugins/ruby/in_kube_events.rb b/source/plugins/ruby/in_kube_events.rb index 6f65dab92..f2d35be7b 100644 --- a/source/plugins/ruby/in_kube_events.rb +++ b/source/plugins/ruby/in_kube_events.rb @@ -18,6 +18,7 @@ def initialize require_relative "oms_common" require_relative "omslog" require_relative "ApplicationInsightsUtility" + require_relative "extension_utils" # refer tomlparser-agent-config for defaults # this configurable via configmap @@ -86,6 +87,13 @@ def enumerate newEventQueryState = [] @eventsCount = 0 + if ExtensionUtils.isAADMSIAuthMode() + $log.info("in_kube_events::enumerate: AAD AUTH MSI MODE") + if !@tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @tag = ExtensionUtils.getOutputStreamId(Constants::KUBE_EVENTS_DATA_TYPE) + end + $log.info("in_kube_events::enumerate: using kubeevents tag -#{@tag} @ #{Time.now.utc.iso8601}") + end # Initializing continuation token to nil continuationToken = nil $log.info("in_kube_events::enumerate : Getting events from Kube API @ #{Time.now.utc.iso8601}") diff --git a/source/plugins/ruby/in_kube_nodes.rb b/source/plugins/ruby/in_kube_nodes.rb index ffc11de55..6018d5f34 100644 --- a/source/plugins/ruby/in_kube_nodes.rb +++ b/source/plugins/ruby/in_kube_nodes.rb @@ -35,7 +35,8 @@ def initialize require_relative "KubernetesApiClient" require_relative "ApplicationInsightsUtility" require_relative "oms_common" - require_relative "omslog" + require_relative "omslog" + require_relative "extension_utils" @ContainerNodeInventoryTag = "oneagent.containerInsights.CONTAINER_NODE_INVENTORY_BLOB" @insightsMetricsTag = "oneagent.containerInsights.INSIGHTS_METRICS_BLOB" @@ -111,6 +112,25 @@ def enumerate @nodeInventoryE2EProcessingLatencyMs = 0 nodeInventoryStartTime = (Time.now.to_f * 1000).to_i + if ExtensionUtils.isAADMSIAuthMode() + $log.info("in_kube_nodes::enumerate: AAD AUTH MSI MODE") + if !@kubeperfTag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @kubeperfTag = ExtensionUtils.getOutputStreamId(Constants::PERF_DATA_TYPE) + end + if !@insightsMetricsTag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @insightsMetricsTag = ExtensionUtils.getOutputStreamId(Constants::INSIGHTS_METRICS_DATA_TYPE) + end + if !@ContainerNodeInventoryTag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @ContainerNodeInventoryTag = ExtensionUtils.getOutputStreamId(Constants::CONTAINER_NODE_INVENTORY_DATA_TYPE) + end + if !@tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @tag = ExtensionUtils.getOutputStreamId(Constants::KUBE_NODE_INVENTORY_DATA_TYPE) + end + $log.info("in_kube_nodes::enumerate: using perf tag -#{@kubeperfTag} @ #{Time.now.utc.iso8601}") + $log.info("in_kube_nodes::enumerate: using insightsmetrics tag -#{@insightsMetricsTag} @ #{Time.now.utc.iso8601}") + $log.info("in_kube_nodes::enumerate: using containernodeinventory tag -#{@ContainerNodeInventoryTag} @ #{Time.now.utc.iso8601}") + $log.info("in_kube_nodes::enumerate: using kubenodeinventory tag -#{@tag} @ #{Time.now.utc.iso8601}") + end nodesAPIChunkStartTime = (Time.now.to_f * 1000).to_i # Initializing continuation token to nil diff --git a/source/plugins/ruby/in_kube_podinventory.rb b/source/plugins/ruby/in_kube_podinventory.rb index 5598602cd..f3a2718e7 100644 --- a/source/plugins/ruby/in_kube_podinventory.rb +++ b/source/plugins/ruby/in_kube_podinventory.rb @@ -27,6 +27,7 @@ def initialize require_relative "oms_common" require_relative "omslog" require_relative "constants" + require_relative "extension_utils" # refer tomlparser-agent-config for updating defaults # this configurable via configmap @@ -108,6 +109,29 @@ def enumerate(podList = nil) serviceRecords = [] @podInventoryE2EProcessingLatencyMs = 0 podInventoryStartTime = (Time.now.to_f * 1000).to_i + if ExtensionUtils.isAADMSIAuthMode() + $log.info("in_kube_podinventory::enumerate: AAD AUTH MSI MODE") + if !@kubeperfTag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @kubeperfTag = ExtensionUtils.getOutputStreamId(Constants::PERF_DATA_TYPE) + end + if !@kubeservicesTag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @kubeservicesTag = ExtensionUtils.getOutputStreamId(Constants::KUBE_SERVICES_DATA_TYPE) + end + if !@containerInventoryTag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @containerInventoryTag = ExtensionUtils.getOutputStreamId(Constants::CONTAINER_INVENTORY_DATA_TYPE) + end + if !@insightsMetricsTag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @insightsMetricsTag = ExtensionUtils.getOutputStreamId(Constants::INSIGHTS_METRICS_DATA_TYPE) + end + if !@tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @tag = ExtensionUtils.getOutputStreamId(Constants::KUBE_POD_INVENTORY_DATA_TYPE) + end + $log.info("in_kube_podinventory::enumerate: using perf tag -#{@kubeperfTag} @ #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::enumerate: using kubeservices tag -#{@kubeservicesTag} @ #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::enumerate: using containerinventory tag -#{@containerInventoryTag} @ #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::enumerate: using insightsmetrics tag -#{@insightsMetricsTag} @ #{Time.now.utc.iso8601}") + $log.info("in_kube_podinventory::enumerate: using kubepodinventory tag -#{@tag} @ #{Time.now.utc.iso8601}") + end # Get services first so that we dont need to make a call for very chunk $log.info("in_kube_podinventory::enumerate : Getting services from Kube API @ #{Time.now.utc.iso8601}") diff --git a/source/plugins/ruby/in_kube_pvinventory.rb b/source/plugins/ruby/in_kube_pvinventory.rb index 40eebac8a..6a95ccd4f 100644 --- a/source/plugins/ruby/in_kube_pvinventory.rb +++ b/source/plugins/ruby/in_kube_pvinventory.rb @@ -20,6 +20,7 @@ def initialize require_relative "oms_common" require_relative "omslog" require_relative "constants" + require_relative "extension_utils" # Response size is around 1500 bytes per PV @PV_CHUNK_SIZE = "5000" @@ -62,6 +63,12 @@ def enumerate @pvTypeToCountHash = {} currentTime = Time.now batchTime = currentTime.utc.iso8601 + if ExtensionUtils.isAADMSIAuthMode() + $log.info("in_kube_pvinventory::enumerate: AAD AUTH MSI MODE") + if !@tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @tag = ExtensionUtils.getOutputStreamId(Constants::KUBE_PV_INVENTORY_DATA_TYPE) + end + end continuationToken = nil $log.info("in_kube_pvinventory::enumerate : Getting PVs from Kube API @ #{Time.now.utc.iso8601}") diff --git a/source/plugins/ruby/in_kubestate_deployments.rb b/source/plugins/ruby/in_kubestate_deployments.rb index 182c3ffc1..dd875862e 100644 --- a/source/plugins/ruby/in_kubestate_deployments.rb +++ b/source/plugins/ruby/in_kubestate_deployments.rb @@ -22,6 +22,7 @@ def initialize require_relative "omslog" require_relative "ApplicationInsightsUtility" require_relative "constants" + require_relative "extension_utils" # refer tomlparser-agent-config for defaults # this configurable via configmap @@ -83,6 +84,12 @@ def enumerate #set the running total for this batch to 0 @deploymentsRunningTotal = 0 + if ExtensionUtils.isAADMSIAuthMode() + $log.info("in_kubestate_deployments::enumerate: AAD AUTH MSI MODE") + if !@tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @tag = ExtensionUtils.getOutputStreamId(Constants::INSIGHTS_METRICS_DATA_TYPE) + end + end # Initializing continuation token to nil continuationToken = nil $log.info("in_kubestate_deployments::enumerate : Getting deployments from Kube API @ #{Time.now.utc.iso8601}") diff --git a/source/plugins/ruby/in_kubestate_hpa.rb b/source/plugins/ruby/in_kubestate_hpa.rb index 8f60bfb72..b1e112f45 100644 --- a/source/plugins/ruby/in_kubestate_hpa.rb +++ b/source/plugins/ruby/in_kubestate_hpa.rb @@ -18,7 +18,8 @@ def initialize require_relative "oms_common" require_relative "omslog" require_relative "ApplicationInsightsUtility" - require_relative "constants" + require_relative "constants" + require_relative "extension_utils" # refer tomlparser-agent-config for defaults # this configurable via configmap @@ -79,6 +80,13 @@ def enumerate @hpaCount = 0 + if ExtensionUtils.isAADMSIAuthMode() + $log.info("in_kubestate_hpa::enumerate: AAD AUTH MSI MODE") + if !@tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @tag = ExtensionUtils.getOutputStreamId(Constants::INSIGHTS_METRICS_DATA_TYPE) + end + $log.info("in_kubestate_hpa::enumerate: using tag -#{@tag} @ #{Time.now.utc.iso8601}") + end # Initializing continuation token to nil continuationToken = nil $log.info("in_kubestate_hpa::enumerate : Getting HPAs from Kube API @ #{Time.now.utc.iso8601}") diff --git a/source/plugins/ruby/in_win_cadvisor_perf.rb b/source/plugins/ruby/in_win_cadvisor_perf.rb index 9ab2474b1..77e84bc2b 100644 --- a/source/plugins/ruby/in_win_cadvisor_perf.rb +++ b/source/plugins/ruby/in_win_cadvisor_perf.rb @@ -20,7 +20,8 @@ def initialize require_relative "oms_common" require_relative "omslog" require_relative "constants" - @insightsMetricsTag = "oneagent.containerInsights.INSIGHTS_METRICS_BLOB" + require_relative "extension_utils" + @insightsMetricsTag = "oneagent.containerInsights.INSIGHTS_METRICS_BLOB" end config_param :run_interval, :time, :default => 60 @@ -58,6 +59,17 @@ def enumerate() timeDifference = (DateTime.now.to_time.to_i - @@winNodeQueryTimeTracker).abs timeDifferenceInMinutes = timeDifference / 60 @@istestvar = ENV["ISTEST"] + if ExtensionUtils.isAADMSIAuthMode() + $log.info("in_win_cadvisor_perf::enumerate: AAD AUTH MSI MODE") + if !@tag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @tag = ExtensionUtils.getOutputStreamId(Constants::PERF_DATA_TYPE) + end + if !@insightsMetricsTag.start_with?(Constants::EXTENSION_OUTPUT_STREAM_ID_TAG_PREFIX) + @insightsMetricsTag = ExtensionUtils.getOutputStreamId(Constants::INSIGHTS_METRICS_DATA_TYPE) + end + $log.info("in_win_cadvisor_perf::enumerate: using perf tag -#{@kubeperfTag} @ #{Time.now.utc.iso8601}") + $log.info("in_win_cadvisor_perf::enumerate: using insightsmetrics tag -#{@insightsMetricsTag} @ #{Time.now.utc.iso8601}") + end #Resetting this cache so that it is populated with the current set of containers with every call CAdvisorMetricsAPIClient.resetWinContainerIdCache() diff --git a/source/plugins/ruby/out_mdm.rb b/source/plugins/ruby/out_mdm.rb index 8e80fb753..247bc01b2 100644 --- a/source/plugins/ruby/out_mdm.rb +++ b/source/plugins/ruby/out_mdm.rb @@ -21,6 +21,9 @@ def initialize require_relative "proxy_utils" @@token_resource_url = "https://monitoring.azure.com/" + # AAD auth supported only in public cloud and handle other clouds when enabled + # this is unified new token audience for LA AAD MSI auth & metrics + @@token_resource_audience = "https://monitor.azure.com/" @@grant_type = "client_credentials" @@azure_json_path = "/etc/kubernetes/host/azure.json" @@post_request_url_template = "https://%{aks_region}.monitoring.azure.com%{aks_resource_id}/metrics" @@ -28,6 +31,8 @@ def initialize # msiEndpoint is the well known endpoint for getting MSI authentications tokens @@msi_endpoint_template = "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&client_id=%{user_assigned_client_id}&resource=%{resource}" + # IMDS msiEndpoint for AAD MSI Auth is the proxy endpoint whcih serves the MSI auth tokens with resource claim + @@imds_msi_endpoint_template = "http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=%{resource}" @@user_assigned_client_id = ENV["USER_ASSIGNED_IDENTITY_CLIENT_ID"] @@plugin_name = "AKSCustomMetricsMDM" @@ -124,7 +129,13 @@ def start @parsed_token_uri = URI.parse(aad_token_url) else @useMsi = true - msi_endpoint = @@msi_endpoint_template % { user_assigned_client_id: @@user_assigned_client_id, resource: @@token_resource_url } + if !@@user_assigned_client_id.nil? && !@@user_assigned_client_id.empty? + msi_endpoint = @@msi_endpoint_template % { user_assigned_client_id: @@user_assigned_client_id, resource: @@token_resource_url } + else + # in case of msi auth user_assigned_client_id will be empty + @log.info "using aad msi auth" + msi_endpoint = @@imds_msi_endpoint_template % { resource: @@token_resource_audience } + end @parsed_token_uri = URI.parse(msi_endpoint) end diff --git a/source/plugins/utils/extension.rb b/source/plugins/utils/extension.rb new file mode 100644 index 000000000..1b1844149 --- /dev/null +++ b/source/plugins/utils/extension.rb @@ -0,0 +1,77 @@ +require "socket" +require "msgpack" +require "securerandom" +require "singleton" +require_relative "omslog" +require_relative "constants" +require_relative "ApplicationInsightsUtility" + + +class Extension + include Singleton + + def initialize + @cache = {} + @cache_lock = Mutex.new + $log.info("Extension::initialize complete") + end + + def get_output_stream_id(datatypeId) + @cache_lock.synchronize { + if @cache.has_key?(datatypeId) + return @cache[datatypeId] + else + @cache = get_config() + return @cache[datatypeId] + end + } + end + + private + def get_config() + extConfig = Hash.new + $log.info("Extension::get_config start ...") + begin + clientSocket = UNIXSocket.open(Constants::ONEAGENT_FLUENT_SOCKET_NAME) + requestId = SecureRandom.uuid.to_s + requestBodyJSON = { "Request" => "AgentTaggedData", "RequestId" => requestId, "Tag" => Constants::CI_EXTENSION_NAME, "Version" => Constants::CI_EXTENSION_VERSION }.to_json + $log.info("sending request with request body: #{requestBodyJSON}") + requestBodyMsgPack = requestBodyJSON.to_msgpack + clientSocket.write(requestBodyMsgPack) + clientSocket.flush + $log.info("reading the response from fluent socket: #{Constants::ONEAGENT_FLUENT_SOCKET_NAME}") + resp = clientSocket.recv(Constants::CI_EXTENSION_CONFIG_MAX_BYTES) + if !resp.nil? && !resp.empty? + $log.info("successfully read the extension config from fluentsocket and number of bytes read is #{resp.length}") + respJSON = JSON.parse(resp) + taggedData = respJSON["TaggedData"] + if !taggedData.nil? && !taggedData.empty? + taggedAgentData = JSON.parse(taggedData) + extensionConfigurations = taggedAgentData["extensionConfigurations"] + if !extensionConfigurations.nil? && !extensionConfigurations.empty? + extensionConfigurations.each do |extensionConfig| + outputStreams = extensionConfig["outputStreams"] + if !outputStreams.nil? && !outputStreams.empty? + outputStreams.each do |datatypeId, streamId| + $log.info("datatypeId: #{datatypeId}, streamId: #{streamId}") + extConfig[datatypeId] = streamId + end + else + $log.info("received outputStreams is either nil or empty") + end + end + else + $log.info("received extensionConfigurations from fluentsocket is either nil or empty") + end + end + end + rescue => errorStr + $log.warn("Extension::get_config failed: #{errorStr}") + ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) + ensure + clientSocket.close unless clientSocket.nil? + end + $log.info("Extension::get_config complete ...") + return extConfig + end +end diff --git a/source/plugins/utils/extension_utils.rb b/source/plugins/utils/extension_utils.rb new file mode 100644 index 000000000..5d439c6b2 --- /dev/null +++ b/source/plugins/utils/extension_utils.rb @@ -0,0 +1,27 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +#!/usr/local/bin/ruby +# frozen_string_literal: true + +require_relative "extension" + +class ExtensionUtils + class << self + def getOutputStreamId(dataType) + outputStreamId = "" + begin + if !dataType.nil? && !dataType.empty? + outputStreamId = Extension.instance.get_output_stream_id(dataType) + $log.info("ExtensionUtils::getOutputStreamId: got streamid: #{outputStreamId} for datatype: #{dataType}") + else + $log.warn("ExtensionUtils::getOutputStreamId: dataType shouldnt be nil or empty") + end + rescue => errorStr + $log.warn("ExtensionUtils::getOutputStreamId: failed with an exception: #{errorStr}") + end + return outputStreamId + end + def isAADMSIAuthMode() + return !ENV["AAD_MSI_AUTH_MODE"].nil? && !ENV["AAD_MSI_AUTH_MODE"].empty? && ENV["AAD_MSI_AUTH_MODE"].downcase == "true" + end + end +end From 679038ba1164814be03f999d4225d63898f3a693 Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Sun, 13 Jun 2021 23:09:52 -0700 Subject: [PATCH 02/14] use existing envvars --- kubernetes/windows/main.ps1 | 11 +---------- source/plugins/go/src/ingestion_token_utils.go | 8 ++++---- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index c28554da9..d0cd2368e 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -61,16 +61,7 @@ function Set-EnvironmentVariables { # Set CLOUD_ENVIRONMENT [System.Environment]::SetEnvironmentVariable("CLOUD_ENVIRONMENT", $cloud_environment, "Process") - [System.Environment]::SetEnvironmentVariable("CLOUD_ENVIRONMENT", $cloud_environment, "Machine") - - - # Set Region - [System.Environment]::SetEnvironmentVariable("customRegion", $env:AKS_REGION, "Process") - [System.Environment]::SetEnvironmentVariable("customRegion", $env:AKS_REGION, "Machine") - - # Set resource ID - [System.Environment]::SetEnvironmentVariable("customResourceId", $env:AKS_RESOURCE_ID, "Process") - [System.Environment]::SetEnvironmentVariable("customResourceId", $env:AKS_RESOURCE_ID, "Machine") + [System.Environment]::SetEnvironmentVariable("CLOUD_ENVIRONMENT", $cloud_environment, "Machine") $wsID = "" if (Test-Path /etc/omsagent-secret/WSID) { diff --git a/source/plugins/go/src/ingestion_token_utils.go b/source/plugins/go/src/ingestion_token_utils.go index 4dffd7974..8b1794e04 100644 --- a/source/plugins/go/src/ingestion_token_utils.go +++ b/source/plugins/go/src/ingestion_token_utils.go @@ -222,8 +222,8 @@ func getAgentConfiguration(imdsAccessToken string) (configurationId string, chan channelId = "" apiVersion := "2020-08-01-preview" var amcs_endpoint *url.URL - resourceId := os.Getenv("customResourceId") - resourceRegion := os.Getenv("customRegion") + resourceId := os.Getenv("AKS_RESOURCE_ID") + resourceRegion := os.Getenv("AKS_REGION") MCS_ENDPOINT := os.Getenv("MCS_ENDPOINT") amcs_endpoint_string := fmt.Sprintf("https://%s.handler.control."+MCS_ENDPOINT+"%s/agentConfigurations?platform=windows&api-version=%s", resourceRegion, resourceId, apiVersion) amcs_endpoint, err = url.Parse(amcs_endpoint_string) @@ -318,8 +318,8 @@ func getIngestionAuthToken(imdsAccessToken string, configurationId string, chann Log("Info getIngestionAuthToken: start") ingestionAuthToken = "" var amcs_endpoint *url.URL - resourceId := os.Getenv("customResourceId") - resourceRegion := os.Getenv("customRegion") + resourceId := os.Getenv("AKS_RESOURCE_ID") + resourceRegion := os.Getenv("AKS_REGION") MCS_ENDPOINT := os.Getenv("MCS_ENDPOINT") apiVersion := "2020-04-01-preview" amcs_endpoint_string := fmt.Sprintf("https://%s.handler.control."+MCS_ENDPOINT+"%s/agentConfigurations/%s/channels/%s/issueIngestionToken?platform=windows&api-version=%s", resourceRegion, resourceId, configurationId, channelId, apiVersion) From b0730e8036879b8997b7944470f9b1c929474b5c Mon Sep 17 00:00:00 2001 From: Ganga Mahesh Siddem Date: Mon, 14 Jun 2021 13:28:20 -0700 Subject: [PATCH 03/14] fix imds token expiry interval --- source/plugins/go/src/ingestion_token_utils.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/plugins/go/src/ingestion_token_utils.go b/source/plugins/go/src/ingestion_token_utils.go index 8b1794e04..36b858722 100644 --- a/source/plugins/go/src/ingestion_token_utils.go +++ b/source/plugins/go/src/ingestion_token_utils.go @@ -95,7 +95,7 @@ func getIngestionToken() (authToken string, err error) { // check if the ingestion token has not been fetched yet or is expiring soon. If so then re-fetch it first. // TODO: re-fetch token in a new thread if it is about to expire (don't block the main thread) - if IMDSToken == "" || IMDSTokenExpiration <= time.Now().Unix()+30 { // refresh the token 30 seconds before it expires + if IMDSToken == "" || IMDSTokenExpiration <= time.Now().Unix()+ 30*60 { // refresh the token 30 minutes before it expires IMDSToken, IMDSTokenExpiration, err = getAccessTokenFromIMDS(true) //TODO: read from an env variable? Or maybe from OS? if err != nil { message := fmt.Sprintf("Error on getAccessTokenFromIMDS %s \n", err.Error()) @@ -116,7 +116,7 @@ func getIngestionToken() (authToken string, err error) { } } - if IngestionAuthToken == "" || IngestionAuthTokenExpiration <= time.Now().Unix()+10 { + if IngestionAuthToken == "" || IngestionAuthTokenExpiration <= time.Now().Unix()+30*60 { IngestionAuthToken, IngestionAuthTokenExpiration, err = getIngestionAuthToken(IMDSToken, ConfigurationId, ChannelId) if err != nil { message := fmt.Sprintf("Error getIngestionAuthToken %s \n", err.Error()) From 43208aabf875d7b6c1dffb696add059b3c3ab8c5 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Thu, 8 Jul 2021 20:17:54 -0700 Subject: [PATCH 04/14] initial proxy support --- kubernetes/linux/main.sh | 11 +++++++++++ kubernetes/linux/setup.sh | 3 ++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index 7950f30fd..7210e945b 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -113,6 +113,7 @@ if [[ ( ( ! -e "/etc/config/kube.conf" ) && ( "${CONTAINER_TYPE}" == "Prometheus fi export PROXY_ENDPOINT="" +# TODO: here # Check for internet connectivity or workspace deletion if [ -e "/etc/omsagent-secret/WSID" ]; then workspaceId=$(cat /etc/omsagent-secret/WSID) @@ -150,6 +151,16 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then else echo "successfully validated provided proxy endpoint is valid and expected format" fi + + # Write environment variables so mdsd can use the proxy. Do this regardless of if using mdsd or not because nothing will break because these environment variables are written + echo $pwd > /opt/proxy_password + + echo "export MDSD_PROXY_MODE=application" >> ~/.bashrc + echo "export MDSD_PROXY_ADDRESS=$hostport" >> ~/.bashrc + echo "export MDSD_PROXY_USERNAME=$user" >> ~/.bashrc + echo "export MDSD_PROXY_PASSWORD_FILE=/opt/proxy_password" >> ~/.bashrc + + source ~/.bashrcaz account set --subscription 3b875bf3-0eec-4d8c-bdee-25c7ccc1f130 fi if [ ! -z "$PROXY_ENDPOINT" ]; then diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 0276ed7af..60c96e1ec 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -10,7 +10,8 @@ sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ update-locale LANG=en_US.UTF-8 #install oneagent - Official bits (05/17/2021) -wget https://github.com/microsoft/Docker-Provider/raw/gangams/ci-aad-auth-msi/oneagent-dev/azure-mdsd_1.10.1-build.dev_x86_64.deb +# wget https://github.com/microsoft/Docker-Provider/raw/gangams/ci-aad-auth-msi/oneagent-dev/azure-mdsd_1.10.1-build.dev_x86_64.deb +wget https://github.com/microsoft/Docker-Provider/releases/download/0.1/azure-mdsd_1.11.0-build.develop.2091_x86_64.deb # has proxy support /usr/bin/dpkg -i $TMPDIR/azure-mdsd*.deb cp -f $TMPDIR/mdsd.xml /etc/mdsd.d From eb99c305c6cfc88a7148b62bfddcccc867911d06 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Mon, 19 Jul 2021 16:27:05 -0700 Subject: [PATCH 05/14] merge? --- kubernetes/linux/main.sh | 6 +- source/plugins/go/src/go.mod | 1 - .../plugins/go/src/ingestion_token_utils.go | 44 +++---- source/plugins/go/src/utils.go | 115 +++++++++--------- 4 files changed, 85 insertions(+), 81 deletions(-) diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index 2a06a931d..bf6073928 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -160,7 +160,11 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then echo "export MDSD_PROXY_USERNAME=$user" >> ~/.bashrc echo "export MDSD_PROXY_PASSWORD_FILE=/opt/proxy_password" >> ~/.bashrc - source ~/.bashrcaz account set --subscription 3b875bf3-0eec-4d8c-bdee-25c7ccc1f130 + # TODO: set proxy in go. It's possible that setting $PROXY_ENDPOINT is good enough, but double check + # go and ruby should automatically use this env variable + echo "export http_proxy=$PROXY_ENDPOINT" >> ~/.bashrc + + source ~/.bashrc fi if [ ! -z "$PROXY_ENDPOINT" ]; then diff --git a/source/plugins/go/src/go.mod b/source/plugins/go/src/go.mod index db29a0553..5fe31dc57 100644 --- a/source/plugins/go/src/go.mod +++ b/source/plugins/go/src/go.mod @@ -31,5 +31,4 @@ require ( k8s.io/api v0.0.0-20180628040859-072894a440bd // indirect k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d k8s.io/client-go v8.0.0+incompatible - golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f ) diff --git a/source/plugins/go/src/ingestion_token_utils.go b/source/plugins/go/src/ingestion_token_utils.go index cad61b474..ba3099f0b 100644 --- a/source/plugins/go/src/ingestion_token_utils.go +++ b/source/plugins/go/src/ingestion_token_utils.go @@ -65,8 +65,8 @@ type AgentConfiguration struct { } `json:"channels"` Extensionconfigurations struct { Containerinsights []struct { - ID string `json:"id"` - Originids []string `json:"originIds"` + ID string `json:"id"` + Originids []string `json:"originIds"` Outputstreams struct { LinuxPerfBlob string `json:"LINUX_PERF_BLOB"` ContainerInventoryBlob string `json:"CONTAINER_INVENTORY_BLOB"` @@ -100,7 +100,7 @@ func getAccessTokenFromIMDS() (string, int64, error) { var responseBytes []byte var err error - if (useIMDSTokenProxyEndPoint != "" && strings.Compare(strings.ToLower(useIMDSTokenProxyEndPoint), "true") == 0) { + if useIMDSTokenProxyEndPoint != "" && strings.Compare(strings.ToLower(useIMDSTokenProxyEndPoint), "true") == 0 { Log("Info Reading IMDS Access Token from IMDS Token proxy endpoint") mcsEndpoint := os.Getenv("MCS_ENDPOINT") msi_endpoint_string := fmt.Sprintf("http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://%s/", mcsEndpoint) @@ -133,14 +133,14 @@ func getAccessTokenFromIMDS() (string, int64, error) { } if resp != nil && resp.Body != nil { - defer resp.Body.Close() + defer resp.Body.Close() } Log("getAccessTokenFromIMDS: IMDS Response Status: %d, retryCount: %d", resp.StatusCode, retryCount) - if IsRetriableError(resp.StatusCode) { + if IsRetriableError(resp.StatusCode) { message := fmt.Sprintf("getAccessTokenFromIMDS: IMDS Request failed with an error code: %d, retryCount: %d", resp.StatusCode, retryCount) Log(message) - retryDelay := time.Duration((retryCount + 1) * 100) * time.Millisecond + retryDelay := time.Duration((retryCount+1)*100) * time.Millisecond if resp.StatusCode == 429 { if resp != nil && resp.Header.Get("Retry-After") != "" { after, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) @@ -150,8 +150,8 @@ func getAccessTokenFromIMDS() (string, int64, error) { } } time.Sleep(retryDelay) + } } - } else { Log("Info Reading IMDS Access Token from file : %s", IMDSTokenPathForWindows) if _, err = os.Stat(IMDSTokenPathForWindows); os.IsNotExist(err) { @@ -163,14 +163,14 @@ func getAccessTokenFromIMDS() (string, int64, error) { responseBytes, err = ioutil.ReadFile(IMDSTokenPathForWindows) if err != nil { Log("getAccessTokenFromIMDS: Could not read IMDS token from file: %s, retryCount: %d", err.Error(), retryCount) - time.Sleep(time.Duration((retryCount + 1) * 100) * time.Millisecond) + time.Sleep(time.Duration((retryCount+1)*100) * time.Millisecond) continue } break - } - } + } + } - if responseBytes == nil { + if responseBytes == nil { Log("getAccessTokenFromIMDS: Error responseBytes is nil") return imdsAccessToken, 0, err } @@ -231,12 +231,12 @@ func getAgentConfiguration(imdsAccessToken string) (configurationId string, chan } if resp != nil && resp.Body != nil { defer resp.Body.Close() - } + } Log("getAgentConfiguration Response Status: %d", resp.StatusCode) if IsRetriableError(resp.StatusCode) { message := fmt.Sprintf("getAgentConfiguration: Request failed with an error code: %d, retryCount: %d", resp.StatusCode, retryCount) Log(message) - retryDelay := time.Duration((retryCount + 1) * 100) * time.Millisecond + retryDelay := time.Duration((retryCount+1)*100) * time.Millisecond if resp.StatusCode == 429 { if resp != nil && resp.Header.Get("Retry-After") != "" { after, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) @@ -329,7 +329,7 @@ func getIngestionAuthToken(imdsAccessToken string, configurationId string, chann req.Header.Add("Authorization", bearer) var resp *http.Response = nil - IsSuccess := false + IsSuccess := false for retryCount := 0; retryCount < MaxRetries; retryCount++ { // Call managed services for Azure resources token endpoint resp, err = HTTPClient.Do(req) @@ -343,13 +343,13 @@ func getIngestionAuthToken(imdsAccessToken string, configurationId string, chann if resp != nil && resp.Body != nil { defer resp.Body.Close() - } + } Log("getIngestionAuthToken Response Status: %d", resp.StatusCode) if IsRetriableError(resp.StatusCode) { message := fmt.Sprintf("getIngestionAuthToken: Request failed with an error code: %d, retryCount: %d", resp.StatusCode, retryCount) Log(message) - retryDelay := time.Duration((retryCount + 1) * 100) * time.Millisecond + retryDelay := time.Duration((retryCount+1)*100) * time.Millisecond if resp.StatusCode == 429 { if resp != nil && resp.Header.Get("Retry-After") != "" { after, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) @@ -357,7 +357,7 @@ func getIngestionAuthToken(imdsAccessToken string, configurationId string, chann retryDelay = time.Duration(after) * time.Second } } - } + } time.Sleep(retryDelay) continue } else if resp.StatusCode != 200 { @@ -432,7 +432,7 @@ func getTokenRefreshIntervalFromAmcsResponse(header http.Header) (refreshInterva func refreshIngestionAuthToken() { for ; true; <-IngestionAuthTokenRefreshTicker.C { - if IMDSToken == "" || IMDSTokenExpiration <= (time.Now().Unix() + 60 * 60) { // token valid 24 hrs and refresh token 1 hr before expiry + if IMDSToken == "" || IMDSTokenExpiration <= (time.Now().Unix()+60*60) { // token valid 24 hrs and refresh token 1 hr before expiry imdsToken, imdsTokenExpiry, err := getAccessTokenFromIMDS() if err != nil { message := fmt.Sprintf("refreshIngestionAuthToken: Error on getAccessTokenFromIMDS %s \n", err.Error()) @@ -460,7 +460,7 @@ func refreshIngestionAuthToken() { continue } } - if IMDSToken == "" || ConfigurationId == "" || ChannelId == "" { + if IMDSToken == "" || ConfigurationId == "" || ChannelId == "" { message := "refreshIngestionAuthToken: IMDSToken or ConfigurationId or ChannelId empty" Log(message) SendException(message) @@ -488,9 +488,9 @@ func refreshIngestionAuthToken() { func IsRetriableError(httpStatusCode int) bool { retryableStatusCodes := [5]int{408, 429, 502, 503, 504} for _, code := range retryableStatusCodes { - if code == httpStatusCode { - return true - } + if code == httpStatusCode { + return true + } } return false } diff --git a/source/plugins/go/src/utils.go b/source/plugins/go/src/utils.go index 02d30607e..b0a8d0ba6 100644 --- a/source/plugins/go/src/utils.go +++ b/source/plugins/go/src/utils.go @@ -12,8 +12,8 @@ import ( "net/url" "os" "strings" - "time" - + "time" + "github.com/Azure/azure-kusto-go/kusto" "github.com/Azure/azure-kusto-go/kusto/ingest" "github.com/Azure/go-autorest/autorest/azure/auth" @@ -87,9 +87,10 @@ func CreateHTTPClient() { } tlsConfig.BuildNameToCertificate() - transport = &http.Transport{TLSClientConfig: tlsConfig} + transport = &http.Transport{TLSClientConfig: tlsConfig} } // set the proxy if the proxy configured + // TODO: read proxy info from secret if ProxyEndpoint != "" { proxyEndpointUrl, err := url.Parse(ProxyEndpoint) if err != nil { @@ -105,7 +106,7 @@ func CreateHTTPClient() { HTTPClient = http.Client{ Transport: transport, Timeout: 30 * time.Second, - } + } Log("Successfully created HTTP Client") } @@ -123,57 +124,57 @@ func ToString(s interface{}) string { //mdsdSocketClient to write msgp messages func CreateMDSDClient(dataType DataType, containerType string) { - mdsdfluentSocket := "/var/run/mdsd/default_fluent.socket" + mdsdfluentSocket := "/var/run/mdsd/default_fluent.socket" if containerType != "" && strings.Compare(strings.ToLower(containerType), "prometheussidecar") == 0 { - mdsdfluentSocket = fmt.Sprintf("/var/run/mdsd-%s/default_fluent.socket", containerType) - } + mdsdfluentSocket = fmt.Sprintf("/var/run/mdsd-%s/default_fluent.socket", containerType) + } switch dataType { - case ContainerLogV2: - if MdsdMsgpUnixSocketClient != nil { - MdsdMsgpUnixSocketClient.Close() - MdsdMsgpUnixSocketClient = nil - } - /*conn, err := fluent.New(fluent.Config{FluentNetwork:"unix", - FluentSocketPath:"/var/run/mdsd/default_fluent.socket", - WriteTimeout: 5 * time.Second, - RequestAck: true}) */ - conn, err := net.DialTimeout("unix", - mdsdfluentSocket, 10*time.Second) - if err != nil { - Log("Error::mdsd::Unable to open MDSD msgp socket connection for ContainerLogV2 %s", err.Error()) - //log.Fatalf("Unable to open MDSD msgp socket connection %s", err.Error()) - } else { - Log("Successfully created MDSD msgp socket connection for ContainerLogV2: %s", mdsdfluentSocket) - MdsdMsgpUnixSocketClient = conn - } - case KubeMonAgentEvents: - if MdsdKubeMonMsgpUnixSocketClient != nil { - MdsdKubeMonMsgpUnixSocketClient.Close() - MdsdKubeMonMsgpUnixSocketClient = nil - } - conn, err := net.DialTimeout("unix", - mdsdfluentSocket, 10*time.Second) - if err != nil { - Log("Error::mdsd::Unable to open MDSD msgp socket connection for KubeMon events %s", err.Error()) - //log.Fatalf("Unable to open MDSD msgp socket connection %s", err.Error()) - } else { - Log("Successfully created MDSD msgp socket connection for KubeMon events:%s", mdsdfluentSocket) - MdsdKubeMonMsgpUnixSocketClient = conn - } - case InsightsMetrics: - if MdsdInsightsMetricsMsgpUnixSocketClient != nil { - MdsdInsightsMetricsMsgpUnixSocketClient.Close() - MdsdInsightsMetricsMsgpUnixSocketClient = nil - } - conn, err := net.DialTimeout("unix", - mdsdfluentSocket, 10*time.Second) - if err != nil { - Log("Error::mdsd::Unable to open MDSD msgp socket connection for insights metrics %s", err.Error()) - //log.Fatalf("Unable to open MDSD msgp socket connection %s", err.Error()) - } else { - Log("Successfully created MDSD msgp socket connection for Insights metrics %s", mdsdfluentSocket) - MdsdInsightsMetricsMsgpUnixSocketClient = conn - } + case ContainerLogV2: + if MdsdMsgpUnixSocketClient != nil { + MdsdMsgpUnixSocketClient.Close() + MdsdMsgpUnixSocketClient = nil + } + /*conn, err := fluent.New(fluent.Config{FluentNetwork:"unix", + FluentSocketPath:"/var/run/mdsd/default_fluent.socket", + WriteTimeout: 5 * time.Second, + RequestAck: true}) */ + conn, err := net.DialTimeout("unix", + mdsdfluentSocket, 10*time.Second) + if err != nil { + Log("Error::mdsd::Unable to open MDSD msgp socket connection for ContainerLogV2 %s", err.Error()) + //log.Fatalf("Unable to open MDSD msgp socket connection %s", err.Error()) + } else { + Log("Successfully created MDSD msgp socket connection for ContainerLogV2: %s", mdsdfluentSocket) + MdsdMsgpUnixSocketClient = conn + } + case KubeMonAgentEvents: + if MdsdKubeMonMsgpUnixSocketClient != nil { + MdsdKubeMonMsgpUnixSocketClient.Close() + MdsdKubeMonMsgpUnixSocketClient = nil + } + conn, err := net.DialTimeout("unix", + mdsdfluentSocket, 10*time.Second) + if err != nil { + Log("Error::mdsd::Unable to open MDSD msgp socket connection for KubeMon events %s", err.Error()) + //log.Fatalf("Unable to open MDSD msgp socket connection %s", err.Error()) + } else { + Log("Successfully created MDSD msgp socket connection for KubeMon events:%s", mdsdfluentSocket) + MdsdKubeMonMsgpUnixSocketClient = conn + } + case InsightsMetrics: + if MdsdInsightsMetricsMsgpUnixSocketClient != nil { + MdsdInsightsMetricsMsgpUnixSocketClient.Close() + MdsdInsightsMetricsMsgpUnixSocketClient = nil + } + conn, err := net.DialTimeout("unix", + mdsdfluentSocket, 10*time.Second) + if err != nil { + Log("Error::mdsd::Unable to open MDSD msgp socket connection for insights metrics %s", err.Error()) + //log.Fatalf("Unable to open MDSD msgp socket connection %s", err.Error()) + } else { + Log("Successfully created MDSD msgp socket connection for Insights metrics %s", mdsdfluentSocket) + MdsdInsightsMetricsMsgpUnixSocketClient = conn + } } } @@ -228,7 +229,7 @@ func isValidUrl(uri string) bool { func convertMsgPackEntriesToMsgpBytes(fluentForwardTag string, msgPackEntries []MsgPackEntry) []byte { var msgpBytes []byte - + fluentForward := MsgPackForward{ Tag: fluentForwardTag, Entries: msgPackEntries, @@ -239,7 +240,7 @@ func convertMsgPackEntriesToMsgpBytes(fluentForwardTag string, msgPackEntries [] msgpSize += 1 + msgp.Int64Size + msgp.GuessSize(fluentForward.Entries[i].Record) } - //allocate buffer for msgp message + //allocate buffer for msgp message msgpBytes = msgp.Require(nil, msgpSize) //construct the stream @@ -252,6 +253,6 @@ func convertMsgPackEntriesToMsgpBytes(fluentForwardTag string, msgPackEntries [] msgpBytes = msgp.AppendInt64(msgpBytes, batchTime) msgpBytes = msgp.AppendMapStrStr(msgpBytes, fluentForward.Entries[entry].Record) } - - return msgpBytes + + return msgpBytes } From c239ff243063d7808d99438e8836fc8cf96ac3ff Mon Sep 17 00:00:00 2001 From: David Michelman Date: Mon, 19 Jul 2021 16:45:26 -0700 Subject: [PATCH 06/14] cleaning up some files which should've merged differently --- kubernetes/omsagent.yaml | 7 - kubernetes/windows/main.ps1 | 16 +- source/plugins/go/src/go.mod | 1 + .../plugins/go/src/ingestion_token_utils.go | 62 ++- .../go/src/ingestion_token_utils_test.go | 54 --- source/plugins/go/src/oms.go | 391 +++++++++--------- source/plugins/go/src/utils.go | 115 +++--- 7 files changed, 295 insertions(+), 351 deletions(-) delete mode 100644 source/plugins/go/src/ingestion_token_utils_test.go diff --git a/kubernetes/omsagent.yaml b/kubernetes/omsagent.yaml index 887ed563b..855f3a8e1 100644 --- a/kubernetes/omsagent.yaml +++ b/kubernetes/omsagent.yaml @@ -811,9 +811,6 @@ spec: - mountPath: C:\etc\config\adx name: omsagent-adx-secret readOnly: true - - mountPath: C:\etc\IMDS-access-token - name: imds-token - readOnly: true # Need to mount this only for airgapped clouds - Commenting this since it wont exist in non airgapped clouds # - mountPath: C:\ca # name: ca-certs @@ -874,10 +871,6 @@ spec: secret: secretName: omsagent-adx-secret optional: true - - name: imds-token - secret: - secretName: omsagent-aad-msi-token - optional: true --- kind: Service apiVersion: v1 diff --git a/kubernetes/windows/main.ps1 b/kubernetes/windows/main.ps1 index bd950b88c..3cbc11e20 100644 --- a/kubernetes/windows/main.ps1 +++ b/kubernetes/windows/main.ps1 @@ -81,10 +81,6 @@ function Set-EnvironmentVariables { # Set DOMAIN [System.Environment]::SetEnvironmentVariable("DOMAIN", $domain, "Process") [System.Environment]::SetEnvironmentVariable("DOMAIN", $domain, "Machine") - - # Set MCS Endpoint - [System.Environment]::SetEnvironmentVariable("MCS_ENDPOINT", $mcs_endpoint, "Process") - [System.Environment]::SetEnvironmentVariable("MCS_ENDPOINT", $mcs_endpoint, "Machine") # Set MCS Endpoint [System.Environment]::SetEnvironmentVariable("MCS_ENDPOINT", $mcs_endpoint, "Process") @@ -92,7 +88,7 @@ function Set-EnvironmentVariables { # Set CLOUD_ENVIRONMENT [System.Environment]::SetEnvironmentVariable("CLOUD_ENVIRONMENT", $cloud_environment, "Process") - [System.Environment]::SetEnvironmentVariable("CLOUD_ENVIRONMENT", $cloud_environment, "Machine") + [System.Environment]::SetEnvironmentVariable("CLOUD_ENVIRONMENT", $cloud_environment, "Machine") $wsID = "" if (Test-Path /etc/omsagent-secret/WSID) { @@ -264,16 +260,6 @@ function Set-EnvironmentVariables { else { Write-Host "Failed to set environment variable HOSTNAME for target 'machine' since it is either null or empty" } - # check if its AAD Auth MSI mode via USING_AAD_MSI_AUTH environment variable - $isAADMSIAuth = [System.Environment]::GetEnvironmentVariable("USING_AAD_MSI_AUTH", "process") - if (![string]::IsNullOrEmpty($isAADMSIAuth) -and ($isAADMSIAuth -eq "true")) { - [System.Environment]::SetEnvironmentVariable("AAD_MSI_AUTH_MODE", "true", "Process") - [System.Environment]::SetEnvironmentVariable("AAD_MSI_AUTH_MODE", "true", "Machine") - Write-Host "Using AAD MSI auth" - } - else { - Write-Host "Using LA Legacy Auth" - } # check if its AAD Auth MSI mode via USING_AAD_MSI_AUTH environment variable $isAADMSIAuth = [System.Environment]::GetEnvironmentVariable("USING_AAD_MSI_AUTH", "process") diff --git a/source/plugins/go/src/go.mod b/source/plugins/go/src/go.mod index 5fe31dc57..db29a0553 100644 --- a/source/plugins/go/src/go.mod +++ b/source/plugins/go/src/go.mod @@ -31,4 +31,5 @@ require ( k8s.io/api v0.0.0-20180628040859-072894a440bd // indirect k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d k8s.io/client-go v8.0.0+incompatible + golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f ) diff --git a/source/plugins/go/src/ingestion_token_utils.go b/source/plugins/go/src/ingestion_token_utils.go index ba3099f0b..c96685042 100644 --- a/source/plugins/go/src/ingestion_token_utils.go +++ b/source/plugins/go/src/ingestion_token_utils.go @@ -65,8 +65,8 @@ type AgentConfiguration struct { } `json:"channels"` Extensionconfigurations struct { Containerinsights []struct { - ID string `json:"id"` - Originids []string `json:"originIds"` + ID string `json:"id"` + Originids []string `json:"originIds"` Outputstreams struct { LinuxPerfBlob string `json:"LINUX_PERF_BLOB"` ContainerInventoryBlob string `json:"CONTAINER_INVENTORY_BLOB"` @@ -100,7 +100,7 @@ func getAccessTokenFromIMDS() (string, int64, error) { var responseBytes []byte var err error - if useIMDSTokenProxyEndPoint != "" && strings.Compare(strings.ToLower(useIMDSTokenProxyEndPoint), "true") == 0 { + if (useIMDSTokenProxyEndPoint != "" && strings.Compare(strings.ToLower(useIMDSTokenProxyEndPoint), "true") == 0) { Log("Info Reading IMDS Access Token from IMDS Token proxy endpoint") mcsEndpoint := os.Getenv("MCS_ENDPOINT") msi_endpoint_string := fmt.Sprintf("http://169.254.169.254/metadata/identity/oauth2/token?api-version=2018-02-01&resource=https://%s/", mcsEndpoint) @@ -133,14 +133,14 @@ func getAccessTokenFromIMDS() (string, int64, error) { } if resp != nil && resp.Body != nil { - defer resp.Body.Close() + defer resp.Body.Close() } Log("getAccessTokenFromIMDS: IMDS Response Status: %d, retryCount: %d", resp.StatusCode, retryCount) - if IsRetriableError(resp.StatusCode) { + if IsRetriableError(resp.StatusCode) { message := fmt.Sprintf("getAccessTokenFromIMDS: IMDS Request failed with an error code: %d, retryCount: %d", resp.StatusCode, retryCount) Log(message) - retryDelay := time.Duration((retryCount+1)*100) * time.Millisecond + retryDelay := time.Duration((retryCount + 1) * 100) * time.Millisecond if resp.StatusCode == 429 { if resp != nil && resp.Header.Get("Retry-After") != "" { after, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) @@ -150,8 +150,28 @@ func getAccessTokenFromIMDS() (string, int64, error) { } } time.Sleep(retryDelay) + continue + } else if resp.StatusCode != 200 { + message := fmt.Sprintf("getAccessTokenFromIMDS: IMDS Request failed with nonretryable error code: %d, retryCount: %d", resp.StatusCode, retryCount) + Log(message) + SendException(message) + return imdsAccessToken, 0, err } + IsSuccess = true + break // call succeeded, don't retry any more + } + if !IsSuccess || resp == nil || resp.Body == nil { + Log("getAccessTokenFromIMDS: IMDS Request ran out of retries") + return imdsAccessToken, 0, err + } + + // Pull out response body + responseBytes, err = ioutil.ReadAll(resp.Body) + if err != nil { + Log("getAccessTokenFromIMDS: Error reading response body: %s", err.Error()) + return imdsAccessToken, 0, err } + } else { Log("Info Reading IMDS Access Token from file : %s", IMDSTokenPathForWindows) if _, err = os.Stat(IMDSTokenPathForWindows); os.IsNotExist(err) { @@ -163,14 +183,14 @@ func getAccessTokenFromIMDS() (string, int64, error) { responseBytes, err = ioutil.ReadFile(IMDSTokenPathForWindows) if err != nil { Log("getAccessTokenFromIMDS: Could not read IMDS token from file: %s, retryCount: %d", err.Error(), retryCount) - time.Sleep(time.Duration((retryCount+1)*100) * time.Millisecond) + time.Sleep(time.Duration((retryCount + 1) * 100) * time.Millisecond) continue } break - } - } + } + } - if responseBytes == nil { + if responseBytes == nil { Log("getAccessTokenFromIMDS: Error responseBytes is nil") return imdsAccessToken, 0, err } @@ -231,12 +251,12 @@ func getAgentConfiguration(imdsAccessToken string) (configurationId string, chan } if resp != nil && resp.Body != nil { defer resp.Body.Close() - } + } Log("getAgentConfiguration Response Status: %d", resp.StatusCode) if IsRetriableError(resp.StatusCode) { message := fmt.Sprintf("getAgentConfiguration: Request failed with an error code: %d, retryCount: %d", resp.StatusCode, retryCount) Log(message) - retryDelay := time.Duration((retryCount+1)*100) * time.Millisecond + retryDelay := time.Duration((retryCount + 1) * 100) * time.Millisecond if resp.StatusCode == 429 { if resp != nil && resp.Header.Get("Retry-After") != "" { after, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) @@ -329,7 +349,7 @@ func getIngestionAuthToken(imdsAccessToken string, configurationId string, chann req.Header.Add("Authorization", bearer) var resp *http.Response = nil - IsSuccess := false + IsSuccess := false for retryCount := 0; retryCount < MaxRetries; retryCount++ { // Call managed services for Azure resources token endpoint resp, err = HTTPClient.Do(req) @@ -343,13 +363,13 @@ func getIngestionAuthToken(imdsAccessToken string, configurationId string, chann if resp != nil && resp.Body != nil { defer resp.Body.Close() - } + } Log("getIngestionAuthToken Response Status: %d", resp.StatusCode) if IsRetriableError(resp.StatusCode) { message := fmt.Sprintf("getIngestionAuthToken: Request failed with an error code: %d, retryCount: %d", resp.StatusCode, retryCount) Log(message) - retryDelay := time.Duration((retryCount+1)*100) * time.Millisecond + retryDelay := time.Duration((retryCount + 1) * 100) * time.Millisecond if resp.StatusCode == 429 { if resp != nil && resp.Header.Get("Retry-After") != "" { after, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) @@ -357,7 +377,7 @@ func getIngestionAuthToken(imdsAccessToken string, configurationId string, chann retryDelay = time.Duration(after) * time.Second } } - } + } time.Sleep(retryDelay) continue } else if resp.StatusCode != 200 { @@ -432,7 +452,7 @@ func getTokenRefreshIntervalFromAmcsResponse(header http.Header) (refreshInterva func refreshIngestionAuthToken() { for ; true; <-IngestionAuthTokenRefreshTicker.C { - if IMDSToken == "" || IMDSTokenExpiration <= (time.Now().Unix()+60*60) { // token valid 24 hrs and refresh token 1 hr before expiry + if IMDSToken == "" || IMDSTokenExpiration <= (time.Now().Unix() + 60 * 60) { // token valid 24 hrs and refresh token 1 hr before expiry imdsToken, imdsTokenExpiry, err := getAccessTokenFromIMDS() if err != nil { message := fmt.Sprintf("refreshIngestionAuthToken: Error on getAccessTokenFromIMDS %s \n", err.Error()) @@ -460,7 +480,7 @@ func refreshIngestionAuthToken() { continue } } - if IMDSToken == "" || ConfigurationId == "" || ChannelId == "" { + if IMDSToken == "" || ConfigurationId == "" || ChannelId == "" { message := "refreshIngestionAuthToken: IMDSToken or ConfigurationId or ChannelId empty" Log(message) SendException(message) @@ -488,9 +508,9 @@ func refreshIngestionAuthToken() { func IsRetriableError(httpStatusCode int) bool { retryableStatusCodes := [5]int{408, 429, 502, 503, 504} for _, code := range retryableStatusCodes { - if code == httpStatusCode { - return true - } + if code == httpStatusCode { + return true + } } return false } diff --git a/source/plugins/go/src/ingestion_token_utils_test.go b/source/plugins/go/src/ingestion_token_utils_test.go deleted file mode 100644 index f987a2408..000000000 --- a/source/plugins/go/src/ingestion_token_utils_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package main - -import ( - "net/http" - "testing" - "time" -) - -// TODO: add unit tests for retry code. - -type headerElementTestType = struct { - in http.Header - want int64 - wantError bool -} - -func TestGetTimeoutFromAmcsResponse(t *testing.T) { - - var cases []headerElementTestType - - cases = append(cases, headerElementTestType{http.Header{ - "Request-Context": {"appId=cid-v1:*******-*********-*********"}, - "Api-Supported-Versions": {"2019-11-02-preview", "2020-04-01-preview", "2020-08-01-preview"}, - "Date": {"Thu, 06 May 2021 21:17:42 GMT"}, - "Cache-Control": {"public", "max-age=3600"}, - "Content-Type": {"application/json; charset=utf-8"}, - "Vary": {"Accept-Encoding"}, - "Server": {"Microsoft-HTTPAPI/2.0"}, - }, 3600, false}) - - cases = append(cases, headerElementTestType{http.Header{ - "Request-Context": {"appId=cid-v1:*******-*********-*********"}, - "Api-Supported-Versions": {"2019-11-02-preview", "2020-04-01-preview", "2020-08-01-preview"}, - "Date": {"Thu, 06 May 2021 21:17:42 GMT"}, - "Cache-Control": {"public", "max-age=300"}, - "Content-Type": {"application/json; charset=utf-8"}, - "Vary": {"Accept-Encoding"}, - "Server": {"Microsoft-HTTPAPI/2.0"}, - }, 300, false}) - - cases = append(cases, headerElementTestType{http.Header{ - "Request-Context": {"appId=cid-v1:*******-*********-*********"}, - }, 0, true}) - - for _, c := range cases { - got, err := getExpirationFromAmcsResponse(c.in) - if (err != nil) != c.wantError { - t.Errorf("getTimeoutFromAmcsResponse() did not return correct error, expected: %v, got %s", c.wantError, err) - } else if err == nil && (got < c.want+time.Now().Unix()-1 || got > c.want+time.Now().Unix()) { - // Adding a range of 1 second should be enough to keep this test from being flaky, but it might have to be re-thought. - t.Errorf("getTimeoutFromAmcsResponse() (%v) == %d, want %d", c.in, got, c.want+time.Now().Unix()) - } - } -} diff --git a/source/plugins/go/src/oms.go b/source/plugins/go/src/oms.go index 438be66d9..0761ef664 100644 --- a/source/plugins/go/src/oms.go +++ b/source/plugins/go/src/oms.go @@ -21,9 +21,8 @@ import ( "github.com/google/uuid" "github.com/tinylib/msgp/msgp" - "Docker-Provider/source/plugins/go/src/extension" - lumberjack "gopkg.in/natefinch/lumberjack.v2" + "Docker-Provider/source/plugins/go/src/extension" "github.com/Azure/azure-kusto-go/kusto/ingest" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -86,6 +85,7 @@ const WindowsContainerLogPluginConfFilePath = "/etc/omsagentwindows/out_oms.conf // IPName const IPName = "ContainerInsights" + const defaultContainerInventoryRefreshInterval = 60 const kubeMonAgentConfigEventFlushInterval = 60 @@ -165,17 +165,17 @@ var ( // ADX tenantID AdxTenantID string //ADX client secret - AdxClientSecret string + AdxClientSecret string // container log or container log v2 tag name for oneagent route - MdsdContainerLogTagName string + MdsdContainerLogTagName string // kubemonagent events tag name for oneagent route MdsdKubeMonAgentEventsTagName string // InsightsMetrics tag name for oneagent route - MdsdInsightsMetricsTagName string + MdsdInsightsMetricsTagName string // flag to check if its Windows OS IsWindows bool - // container type - ContainerType string + // container type + ContainerType string // flag to check whether LA AAD MSI Auth Enabled or not IsAADMSIAuthMode bool ) @@ -206,7 +206,7 @@ var ( // IngestionAuthTokenUpdateMutex read and write mutex access for ODSIngestionAuthToken IngestionAuthTokenUpdateMutex = &sync.Mutex{} // ODSIngestionAuthToken for windows agent AAD MSI Auth - ODSIngestionAuthToken string + ODSIngestionAuthToken string ) var ( @@ -247,29 +247,29 @@ type DataItemLAv1 struct { // DataItemLAv2 == ContainerLogV2 table in LA // Please keep the names same as destination column names, to avoid transforming one to another in the pipeline type DataItemLAv2 struct { - TimeGenerated string `json:"TimeGenerated"` - Computer string `json:"Computer"` - ContainerId string `json:"ContainerId"` - ContainerName string `json:"ContainerName"` - PodName string `json:"PodName"` - PodNamespace string `json:"PodNamespace"` - LogMessage string `json:"LogMessage"` - LogSource string `json:"LogSource"` + TimeGenerated string `json:"TimeGenerated"` + Computer string `json:"Computer"` + ContainerId string `json:"ContainerId"` + ContainerName string `json:"ContainerName"` + PodName string `json:"PodName"` + PodNamespace string `json:"PodNamespace"` + LogMessage string `json:"LogMessage"` + LogSource string `json:"LogSource"` //PodLabels string `json:"PodLabels"` } // DataItemADX == ContainerLogV2 table in ADX type DataItemADX struct { - TimeGenerated string `json:"TimeGenerated"` - Computer string `json:"Computer"` - ContainerId string `json:"ContainerId"` - ContainerName string `json:"ContainerName"` - PodName string `json:"PodName"` - PodNamespace string `json:"PodNamespace"` - LogMessage string `json:"LogMessage"` - LogSource string `json:"LogSource"` + TimeGenerated string `json:"TimeGenerated"` + Computer string `json:"Computer"` + ContainerId string `json:"ContainerId"` + ContainerName string `json:"ContainerName"` + PodName string `json:"PodName"` + PodNamespace string `json:"PodNamespace"` + LogMessage string `json:"LogMessage"` + LogSource string `json:"LogSource"` //PodLabels string `json:"PodLabels"` - AzureResourceId string `json:"AzureResourceId"` + AzureResourceId string `json:"AzureResourceId"` } // telegraf metric DataItem represents the object corresponding to the json that is sent by fluentbit tail plugin @@ -294,15 +294,15 @@ type InsightsMetricsBlob struct { // ContainerLogBlob represents the object corresponding to the payload that is sent to the ODS end point type ContainerLogBlobLAv1 struct { - DataType string `json:"DataType"` - IPName string `json:"IPName"` + DataType string `json:"DataType"` + IPName string `json:"IPName"` DataItems []DataItemLAv1 `json:"DataItems"` } // ContainerLogBlob represents the object corresponding to the payload that is sent to the ODS end point type ContainerLogBlobLAv2 struct { - DataType string `json:"DataType"` - IPName string `json:"IPName"` + DataType string `json:"DataType"` + IPName string `json:"IPName"` DataItems []DataItemLAv2 `json:"DataItems"` } @@ -355,13 +355,12 @@ const ( ) // DataType to be used as enum per data type socket client creation -type DataType int - +type DataType int const ( // DataType to be used as enum per data type socket client creation ContainerLogV2 DataType = iota - KubeMonAgentEvents - InsightsMetrics + KubeMonAgentEvents + InsightsMetrics ) func createLogger() *log.Logger { @@ -609,7 +608,7 @@ func flushKubeMonAgentEventRecords() { Message: k, Tags: fmt.Sprintf("%s", tagJson), } - laKubeMonAgentEventsRecords = append(laKubeMonAgentEventsRecords, laKubeMonAgentEventsRecord) + laKubeMonAgentEventsRecords = append(laKubeMonAgentEventsRecords, laKubeMonAgentEventsRecord) var stringMap map[string]string jsonBytes, err := json.Marshal(&laKubeMonAgentEventsRecord) if err != nil { @@ -622,12 +621,12 @@ func flushKubeMonAgentEventRecords() { Log(message) SendException(message) } else { - msgPackEntry := MsgPackEntry{ + msgPackEntry := MsgPackEntry{ Record: stringMap, } - msgPackEntries = append(msgPackEntries, msgPackEntry) - } - } + msgPackEntries = append(msgPackEntries, msgPackEntry) + } + } } } @@ -648,24 +647,24 @@ func flushKubeMonAgentEventRecords() { Message: k, Tags: fmt.Sprintf("%s", tagJson), } - laKubeMonAgentEventsRecords = append(laKubeMonAgentEventsRecords, laKubeMonAgentEventsRecord) + laKubeMonAgentEventsRecords = append(laKubeMonAgentEventsRecords, laKubeMonAgentEventsRecord) var stringMap map[string]string jsonBytes, err := json.Marshal(&laKubeMonAgentEventsRecord) if err != nil { message := fmt.Sprintf("Error while Marshalling laKubeMonAgentEventsRecord to json bytes: %s", err.Error()) Log(message) SendException(message) - } else { - if err := json.Unmarshal(jsonBytes, &stringMap); err != nil { + } else { + if err := json.Unmarshal(jsonBytes, &stringMap); err != nil { message := fmt.Sprintf("Error while UnMarhalling json bytes to stringmap: %s", err.Error()) Log(message) SendException(message) } else { - msgPackEntry := MsgPackEntry{ + msgPackEntry := MsgPackEntry{ Record: stringMap, - } - msgPackEntries = append(msgPackEntries, msgPackEntry) - } + } + msgPackEntries = append(msgPackEntries, msgPackEntry) + } } } } @@ -697,66 +696,66 @@ func flushKubeMonAgentEventRecords() { Message: "No errors", Tags: fmt.Sprintf("%s", tagJson), } - laKubeMonAgentEventsRecords = append(laKubeMonAgentEventsRecords, laKubeMonAgentEventsRecord) + laKubeMonAgentEventsRecords = append(laKubeMonAgentEventsRecords, laKubeMonAgentEventsRecord) var stringMap map[string]string jsonBytes, err := json.Marshal(&laKubeMonAgentEventsRecord) - if err != nil { + if err != nil { message := fmt.Sprintf("Error while Marshalling laKubeMonAgentEventsRecord to json bytes: %s", err.Error()) Log(message) SendException(message) } else { - if err := json.Unmarshal(jsonBytes, &stringMap); err != nil { + if err := json.Unmarshal(jsonBytes, &stringMap); err != nil { message := fmt.Sprintf("Error while UnMarshalling json bytes to stringmap: %s", err.Error()) - Log(message) - SendException(message) - } else { - msgPackEntry := MsgPackEntry{ + Log(message) + SendException(message) + } else { + msgPackEntry := MsgPackEntry{ Record: stringMap, - } - msgPackEntries = append(msgPackEntries, msgPackEntry) + } + msgPackEntries = append(msgPackEntries, msgPackEntry) } } } } - if IsWindows == false && len(msgPackEntries) > 0 { //for linux, mdsd route + if (IsWindows == false && len(msgPackEntries) > 0) { //for linux, mdsd route if IsAADMSIAuthMode == true && strings.HasPrefix(MdsdKubeMonAgentEventsTagName, MdsdOutputStreamIdTagPrefix) == false { Log("Info::mdsd::obtaining output stream id for data type: %s", KubeMonAgentEventDataType) MdsdKubeMonAgentEventsTagName = extension.GetInstance(FLBLogger, ContainerType).GetOutputStreamId(KubeMonAgentEventDataType) - } + } Log("Info::mdsd:: using mdsdsource name for KubeMonAgentEvents: %s", MdsdKubeMonAgentEventsTagName) - msgpBytes := convertMsgPackEntriesToMsgpBytes(MdsdKubeMonAgentEventsTagName, msgPackEntries) + msgpBytes := convertMsgPackEntriesToMsgpBytes(MdsdKubeMonAgentEventsTagName, msgPackEntries) if MdsdKubeMonMsgpUnixSocketClient == nil { Log("Error::mdsd::mdsd connection for KubeMonAgentEvents does not exist. re-connecting ...") CreateMDSDClient(KubeMonAgentEvents, ContainerType) if MdsdKubeMonMsgpUnixSocketClient == nil { - Log("Error::mdsd::Unable to create mdsd client for KubeMonAgentEvents. Please check error log.") + Log("Error::mdsd::Unable to create mdsd client for KubeMonAgentEvents. Please check error log.") ContainerLogTelemetryMutex.Lock() defer ContainerLogTelemetryMutex.Unlock() - KubeMonEventsMDSDClientCreateErrors += 1 - } + KubeMonEventsMDSDClientCreateErrors += 1 + } } - if MdsdKubeMonMsgpUnixSocketClient != nil { + if MdsdKubeMonMsgpUnixSocketClient != nil { deadline := 10 * time.Second - MdsdKubeMonMsgpUnixSocketClient.SetWriteDeadline(time.Now().Add(deadline)) //this is based of clock time, so cannot reuse + MdsdKubeMonMsgpUnixSocketClient.SetWriteDeadline(time.Now().Add(deadline)) //this is based of clock time, so cannot reuse bts, er := MdsdKubeMonMsgpUnixSocketClient.Write(msgpBytes) - elapsed = time.Since(start) + elapsed = time.Since(start) if er != nil { message := fmt.Sprintf("Error::mdsd::Failed to write to kubemonagent mdsd %d records after %s. Will retry ... error : %s", len(msgPackEntries), elapsed, er.Error()) Log(message) if MdsdKubeMonMsgpUnixSocketClient != nil { MdsdKubeMonMsgpUnixSocketClient.Close() MdsdKubeMonMsgpUnixSocketClient = nil - } + } SendException(message) } else { numRecords := len(msgPackEntries) Log("FlushKubeMonAgentEventRecords::Info::Successfully flushed %d records that was %d bytes in %s", numRecords, bts, elapsed) - // Send telemetry to AppInsights resource + // Send telemetry to AppInsights resource SendEvent(KubeMonAgentEventsFlushedEvent, telemetryDimensions) - } + } } else { - Log("Error::mdsd::Unable to create mdsd client for KubeMonAgentEvents. Please check error log.") - } + Log("Error::mdsd::Unable to create mdsd client for KubeMonAgentEvents. Please check error log.") + } } else if len(laKubeMonAgentEventsRecords) > 0 { //for windows, ODS direct kubeMonAgentEventEntry := KubeMonAgentEventBlob{ DataType: KubeMonAgentEventDataType, @@ -782,11 +781,11 @@ func flushKubeMonAgentEventRecords() { if IsAADMSIAuthMode == true { IngestionAuthTokenUpdateMutex.Lock() - ingestionAuthToken := ODSIngestionAuthToken - IngestionAuthTokenUpdateMutex.Unlock() - if ingestionAuthToken == "" { - Log("Error::ODS Ingestion Auth Token is empty. Please check error log.") - } + ingestionAuthToken := ODSIngestionAuthToken + IngestionAuthTokenUpdateMutex.Unlock() + if ingestionAuthToken == "" { + Log("Error::ODS Ingestion Auth Token is empty. Please check error log.") + } req.Header.Set("Authorization", "Bearer "+ingestionAuthToken) } @@ -899,82 +898,82 @@ func PostTelegrafMetricsToLA(telegrafRecords []map[interface{}]interface{}) int message := fmt.Sprintf("PostTelegrafMetricsToLA::Info:derived %v metrics from %v timeseries", len(laMetrics), len(telegrafRecords)) Log(message) } - + if IsWindows == false { //for linux, mdsd route - var msgPackEntries []MsgPackEntry + var msgPackEntries []MsgPackEntry var i int start := time.Now() - var elapsed time.Duration + var elapsed time.Duration - for i = 0; i < len(laMetrics); i++ { - var interfaceMap map[string]interface{} - stringMap := make(map[string]string) - jsonBytes, err := json.Marshal(*laMetrics[i]) - if err != nil { - message := fmt.Sprintf("PostTelegrafMetricsToLA::Error:when marshalling json %q", err) - Log(message) - SendException(message) - return output.FLB_OK - } else { - if err := json.Unmarshal(jsonBytes, &interfaceMap); err != nil { - message := fmt.Sprintf("Error while UnMarshalling json bytes to interfaceMap: %s", err.Error()) + for i = 0; i < len(laMetrics); i++ { + var interfaceMap map[string]interface{} + stringMap := make(map[string]string) + jsonBytes, err := json.Marshal(*laMetrics[i]) + if err != nil { + message := fmt.Sprintf("PostTelegrafMetricsToLA::Error:when marshalling json %q", err) Log(message) SendException(message) return output.FLB_OK } else { - for key, value := range interfaceMap { - strKey := fmt.Sprintf("%v", key) - strValue := fmt.Sprintf("%v", value) - stringMap[strKey] = strValue - } - msgPackEntry := MsgPackEntry{ - Record: stringMap, - } - msgPackEntries = append(msgPackEntries, msgPackEntry) + if err := json.Unmarshal(jsonBytes, &interfaceMap); err != nil { + message := fmt.Sprintf("Error while UnMarshalling json bytes to interfaceMap: %s", err.Error()) + Log(message) + SendException(message) + return output.FLB_OK + } else { + for key, value := range interfaceMap { + strKey := fmt.Sprintf("%v", key) + strValue := fmt.Sprintf("%v", value) + stringMap[strKey] = strValue + } + msgPackEntry := MsgPackEntry{ + Record: stringMap, + } + msgPackEntries = append(msgPackEntries, msgPackEntry) + } } - } } - if len(msgPackEntries) > 0 { - if IsAADMSIAuthMode == true && (strings.HasPrefix(MdsdInsightsMetricsTagName, MdsdOutputStreamIdTagPrefix) == false) { - Log("Info::mdsd::obtaining output stream id for InsightsMetricsDataType since Log Analytics AAD MSI Auth Enabled") - MdsdInsightsMetricsTagName = extension.GetInstance(FLBLogger, ContainerType).GetOutputStreamId(InsightsMetricsDataType) - } - msgpBytes := convertMsgPackEntriesToMsgpBytes(MdsdInsightsMetricsTagName, msgPackEntries) - if MdsdInsightsMetricsMsgpUnixSocketClient == nil { - Log("Error::mdsd::mdsd connection does not exist. re-connecting ...") - CreateMDSDClient(InsightsMetrics, ContainerType) + if (len(msgPackEntries) > 0) { + if IsAADMSIAuthMode == true && (strings.HasPrefix(MdsdInsightsMetricsTagName, MdsdOutputStreamIdTagPrefix) == false) { + Log("Info::mdsd::obtaining output stream id for InsightsMetricsDataType since Log Analytics AAD MSI Auth Enabled") + MdsdInsightsMetricsTagName = extension.GetInstance(FLBLogger, ContainerType).GetOutputStreamId(InsightsMetricsDataType) + } + msgpBytes := convertMsgPackEntriesToMsgpBytes(MdsdInsightsMetricsTagName, msgPackEntries) if MdsdInsightsMetricsMsgpUnixSocketClient == nil { - Log("Error::mdsd::Unable to create mdsd client for insights metrics. Please check error log.") - ContainerLogTelemetryMutex.Lock() - defer ContainerLogTelemetryMutex.Unlock() - InsightsMetricsMDSDClientCreateErrors += 1 - return output.FLB_RETRY + Log("Error::mdsd::mdsd connection does not exist. re-connecting ...") + CreateMDSDClient(InsightsMetrics, ContainerType) + if MdsdInsightsMetricsMsgpUnixSocketClient == nil { + Log("Error::mdsd::Unable to create mdsd client for insights metrics. Please check error log.") + ContainerLogTelemetryMutex.Lock() + defer ContainerLogTelemetryMutex.Unlock() + InsightsMetricsMDSDClientCreateErrors += 1 + return output.FLB_RETRY + } } - } - deadline := 10 * time.Second - MdsdInsightsMetricsMsgpUnixSocketClient.SetWriteDeadline(time.Now().Add(deadline)) //this is based of clock time, so cannot reuse - bts, er := MdsdInsightsMetricsMsgpUnixSocketClient.Write(msgpBytes) + deadline := 10 * time.Second + MdsdInsightsMetricsMsgpUnixSocketClient.SetWriteDeadline(time.Now().Add(deadline)) //this is based of clock time, so cannot reuse + bts, er := MdsdInsightsMetricsMsgpUnixSocketClient.Write(msgpBytes) - elapsed = time.Since(start) + elapsed = time.Since(start) - if er != nil { - Log("Error::mdsd::Failed to write to mdsd %d records after %s. Will retry ... error : %s", len(msgPackEntries), elapsed, er.Error()) - if MdsdInsightsMetricsMsgpUnixSocketClient != nil { - MdsdInsightsMetricsMsgpUnixSocketClient.Close() - MdsdInsightsMetricsMsgpUnixSocketClient = nil - } + if er != nil { + Log("Error::mdsd::Failed to write to mdsd %d records after %s. Will retry ... error : %s", len(msgPackEntries), elapsed, er.Error()) + if MdsdInsightsMetricsMsgpUnixSocketClient != nil { + MdsdInsightsMetricsMsgpUnixSocketClient.Close() + MdsdInsightsMetricsMsgpUnixSocketClient = nil + } - ContainerLogTelemetryMutex.Lock() - defer ContainerLogTelemetryMutex.Unlock() - InsightsMetricsMDSDClientCreateErrors += 1 - return output.FLB_RETRY - } else { - numTelegrafMetricsRecords := len(msgPackEntries) - Log("Success::mdsd::Successfully flushed %d telegraf metrics records that was %d bytes to mdsd in %s ", numTelegrafMetricsRecords, bts, elapsed) - } + ContainerLogTelemetryMutex.Lock() + defer ContainerLogTelemetryMutex.Unlock() + InsightsMetricsMDSDClientCreateErrors += 1 + return output.FLB_RETRY + } else { + numTelegrafMetricsRecords := len(msgPackEntries) + Log("Success::mdsd::Successfully flushed %d telegraf metrics records that was %d bytes to mdsd in %s ", numTelegrafMetricsRecords, bts, elapsed) + } } - + } else { // for windows, ODS direct var metrics []laTelegrafMetric @@ -1016,9 +1015,9 @@ func PostTelegrafMetricsToLA(telegrafRecords []map[interface{}]interface{}) int if IsAADMSIAuthMode == true { IngestionAuthTokenUpdateMutex.Lock() ingestionAuthToken := ODSIngestionAuthToken - IngestionAuthTokenUpdateMutex.Unlock() - if ingestionAuthToken == "" { - message := "Error::ODS Ingestion Auth Token is empty. Please check error log." + IngestionAuthTokenUpdateMutex.Unlock() + if ingestionAuthToken == "" { + message := "Error::ODS Ingestion Auth Token is empty. Please check error log." Log(message) return output.FLB_RETRY } @@ -1109,12 +1108,12 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { stringMap = make(map[string]string) //below id & name are used by latency telemetry in both v1 & v2 LA schemas id := "" - name := "" + name := "" logEntry := ToString(record["log"]) logEntryTimeStamp := ToString(record["time"]) //ADX Schema & LAv2 schema are almost the same (except resourceId) - if ContainerLogSchemaV2 == true || ContainerLogsRouteADX == true { + if (ContainerLogSchemaV2 == true || ContainerLogsRouteADX == true) { stringMap["Computer"] = Computer stringMap["ContainerId"] = containerID stringMap["ContainerName"] = containerName @@ -1163,29 +1162,29 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { stringMap["AzureResourceId"] = "" } dataItemADX = DataItemADX{ - TimeGenerated: stringMap["TimeGenerated"], - Computer: stringMap["Computer"], - ContainerId: stringMap["ContainerId"], - ContainerName: stringMap["ContainerName"], - PodName: stringMap["PodName"], - PodNamespace: stringMap["PodNamespace"], - LogMessage: stringMap["LogMessage"], - LogSource: stringMap["LogSource"], - AzureResourceId: stringMap["AzureResourceId"], + TimeGenerated: stringMap["TimeGenerated"], + Computer: stringMap["Computer"], + ContainerId: stringMap["ContainerId"], + ContainerName: stringMap["ContainerName"], + PodName: stringMap["PodName"], + PodNamespace: stringMap["PodNamespace"], + LogMessage: stringMap["LogMessage"], + LogSource: stringMap["LogSource"], + AzureResourceId: stringMap["AzureResourceId"], } //ADX dataItemsADX = append(dataItemsADX, dataItemADX) } else { - if ContainerLogSchemaV2 == true { + if (ContainerLogSchemaV2 == true) { dataItemLAv2 = DataItemLAv2{ - TimeGenerated: stringMap["TimeGenerated"], - Computer: stringMap["Computer"], - ContainerId: stringMap["ContainerId"], - ContainerName: stringMap["ContainerName"], - PodName: stringMap["PodName"], - PodNamespace: stringMap["PodNamespace"], - LogMessage: stringMap["LogMessage"], - LogSource: stringMap["LogSource"], + TimeGenerated: stringMap["TimeGenerated"], + Computer: stringMap["Computer"], + ContainerId: stringMap["ContainerId"], + ContainerName: stringMap["ContainerName"], + PodName: stringMap["PodName"], + PodNamespace: stringMap["PodNamespace"], + LogMessage: stringMap["LogMessage"], + LogSource: stringMap["LogSource"], } //ODS-v2 schema dataItemsLAv2 = append(dataItemsLAv2, dataItemLAv2) @@ -1203,10 +1202,10 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { Image: stringMap["Image"], Name: stringMap["Name"], } - //ODS-v1 schema - dataItemsLAv1 = append(dataItemsLAv1, dataItemLAv1) - name = stringMap["Name"] - id = stringMap["Id"] + //ODS-v1 schema + dataItemsLAv1 = append(dataItemsLAv1, dataItemLAv1) + name = stringMap["Name"] + id = stringMap["Id"] } } @@ -1229,7 +1228,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { numContainerLogRecords := 0 if len(msgPackEntries) > 0 && ContainerLogsRouteV2 == true { - //flush to mdsd + //flush to mdsd if IsAADMSIAuthMode == true && strings.HasPrefix(MdsdContainerLogTagName, MdsdOutputStreamIdTagPrefix) == false { Log("Info::mdsd::obtaining output stream id") if ContainerLogSchemaV2 == true { @@ -1239,7 +1238,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { } Log("Info::mdsd:: using mdsdsource name: %s", MdsdContainerLogTagName) } - + fluentForward := MsgPackForward{ Tag: MdsdContainerLogTagName, Entries: msgPackEntries, @@ -1361,13 +1360,13 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { recordType := "" loglinesCount := 0 //schema v2 - if len(dataItemsLAv2) > 0 && ContainerLogSchemaV2 == true { + if (len(dataItemsLAv2) > 0 && ContainerLogSchemaV2 == true) { logEntry = ContainerLogBlobLAv2{ DataType: ContainerLogV2DataType, IPName: IPName, DataItems: dataItemsLAv2} - loglinesCount = len(dataItemsLAv2) - recordType = "ContainerLogV2" + loglinesCount = len(dataItemsLAv2) + recordType = "ContainerLogV2" } else { //schema v1 if len(dataItemsLAv1) > 0 { @@ -1375,8 +1374,8 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { DataType: ContainerLogDataType, IPName: IPName, DataItems: dataItemsLAv1} - loglinesCount = len(dataItemsLAv1) - recordType = "ContainerLog" + loglinesCount = len(dataItemsLAv1) + recordType = "ContainerLog" } } @@ -1398,19 +1397,19 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { if ResourceCentric == true { req.Header.Set("x-ms-AzureResourceId", ResourceID) } - + if IsAADMSIAuthMode == true { IngestionAuthTokenUpdateMutex.Lock() ingestionAuthToken := ODSIngestionAuthToken IngestionAuthTokenUpdateMutex.Unlock() - if ingestionAuthToken == "" { - Log("Error::ODS Ingestion Auth Token is empty. Please check error log.") + if ingestionAuthToken == "" { + Log("Error::ODS Ingestion Auth Token is empty. Please check error log.") return output.FLB_RETRY } // add authorization header to the req - req.Header.Set("Authorization", "Bearer "+ingestionAuthToken) - } - + req.Header.Set("Authorization", "Bearer "+ingestionAuthToken) + } + resp, err := HTTPClient.Do(req) elapsed = time.Since(start) @@ -1419,7 +1418,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { Log(message) // Commenting this out for now. TODO - Add better telemetry for ods errors using aggregation //SendException(message) - + Log("Failed to flush %d records after %s", loglinesCount, elapsed) return output.FLB_RETRY @@ -1436,7 +1435,7 @@ func PostDataHelper(tailPluginRecords []map[interface{}]interface{}) int { numContainerLogRecords = loglinesCount Log("PostDataHelper::Info::Successfully flushed %d %s records to ODS in %s", numContainerLogRecords, recordType, elapsed) - } + } ContainerLogTelemetryMutex.Lock() defer ContainerLogTelemetryMutex.Unlock() @@ -1507,7 +1506,7 @@ func GetContainerIDK8sNamespacePodNameFromFileName(filename string) (string, str } // InitializePlugin reads and populates plugin configuration -func InitializePlugin(pluginConfPath string, agentVersion string) { +func InitializePlugin(pluginConfPath string, agentVersion string) { go func() { isTest := os.Getenv("ISTEST") if strings.Compare(strings.ToLower(strings.TrimSpace(isTest)), "true") == 0 { @@ -1547,10 +1546,10 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { } ContainerType = os.Getenv(ContainerTypeEnv) - Log("Container Type %s", ContainerType) + Log("Container Type %s", ContainerType) osType := os.Getenv("OS_TYPE") - IsWindows = false + IsWindows = false // Linux if strings.Compare(strings.ToLower(osType), "windows") != 0 { Log("Reading configuration for Linux from %s", pluginConfPath) @@ -1569,7 +1568,7 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { SendException(message) time.Sleep(30 * time.Second) log.Fatalln(message) - } + } OMSEndpoint = "https://" + WorkspaceID + ".ods." + LogAnalyticsWorkspaceDomain + "/OperationalData.svc/PostJsonDataItems" // Populate Computer field containerHostName, err1 := ioutil.ReadFile(pluginConfig["container_host_file_path"]) @@ -1599,7 +1598,7 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { } } else { // windows - IsWindows = true + IsWindows = true Computer = os.Getenv("HOSTNAME") WorkspaceID = os.Getenv("WSID") logAnalyticsDomain := os.Getenv("DOMAIN") @@ -1611,7 +1610,7 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { IsAADMSIAuthMode = false if strings.Compare(strings.ToLower(os.Getenv(AADMSIAuthMode)), "true") == 0 { IsAADMSIAuthMode = true - Log("AAD MSI Auth Mode Configured") + Log("AAD MSI Auth Mode Configured") } ResourceID = os.Getenv(envAKSResourceID) @@ -1686,13 +1685,13 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { Log(message) } - PluginConfiguration = pluginConfig + PluginConfiguration = pluginConfig ContainerLogsRoute := strings.TrimSpace(strings.ToLower(os.Getenv("AZMON_CONTAINER_LOGS_ROUTE"))) Log("AZMON_CONTAINER_LOGS_ROUTE:%s", ContainerLogsRoute) - ContainerLogsRouteV2 = false - ContainerLogsRouteADX = false + ContainerLogsRouteV2 = false + ContainerLogsRouteADX = false if strings.Compare(ContainerLogsRoute, ContainerLogsADXRoute) == 0 { //check if adx clusteruri, clientid & secret are set @@ -1725,14 +1724,14 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { Log("Routing container logs thru %s route...", ContainerLogsADXRoute) fmt.Fprintf(os.Stdout, "Routing container logs thru %s route...\n", ContainerLogsADXRoute) } - } else if strings.Compare(strings.ToLower(osType), "windows") != 0 { //for linux, oneagent will be default route - ContainerLogsRouteV2 = true //default is mdsd route - if strings.Compare(ContainerLogsRoute, ContainerLogsV1Route) == 0 { - ContainerLogsRouteV2 = false //fallback option when hiddensetting set + } else if strings.Compare(strings.ToLower(osType), "windows") != 0 { //for linux, oneagent will be default route + ContainerLogsRouteV2 = true //default is mdsd route + if strings.Compare(ContainerLogsRoute, ContainerLogsV1Route) == 0 { + ContainerLogsRouteV2 = false //fallback option when hiddensetting set } Log("Routing container logs thru %s route...", ContainerLogsRoute) fmt.Fprintf(os.Stdout, "Routing container logs thru %s route... \n", ContainerLogsRoute) - } + } if ContainerLogsRouteV2 == true { CreateMDSDClient(ContainerLogV2, ContainerType) @@ -1745,16 +1744,16 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { if IsWindows == false { // mdsd linux specific Log("Creating MDSD clients for KubeMonAgentEvents & InsightsMetrics") - CreateMDSDClient(KubeMonAgentEvents, ContainerType) + CreateMDSDClient(KubeMonAgentEvents, ContainerType) CreateMDSDClient(InsightsMetrics, ContainerType) - } + } ContainerLogSchemaVersion := strings.TrimSpace(strings.ToLower(os.Getenv("AZMON_CONTAINER_LOG_SCHEMA_VERSION"))) Log("AZMON_CONTAINER_LOG_SCHEMA_VERSION:%s", ContainerLogSchemaVersion) - ContainerLogSchemaV2 = false //default is v1 schema + ContainerLogSchemaV2 = false //default is v1 schema - if strings.Compare(ContainerLogSchemaVersion, ContainerLogV2SchemaVersion) == 0 && ContainerLogsRouteADX != true { + if strings.Compare(ContainerLogSchemaVersion, ContainerLogV2SchemaVersion) == 0 && ContainerLogsRouteADX != true { ContainerLogSchemaV2 = true Log("Container logs schema=%s", ContainerLogV2SchemaVersion) fmt.Fprintf(os.Stdout, "Container logs schema=%s... \n", ContainerLogV2SchemaVersion) @@ -1780,15 +1779,15 @@ func InitializePlugin(pluginConfPath string, agentVersion string) { if ContainerLogSchemaV2 == true { MdsdContainerLogTagName = MdsdContainerLogV2SourceName } else { - MdsdContainerLogTagName = MdsdContainerLogSourceName - } + MdsdContainerLogTagName = MdsdContainerLogSourceName + } MdsdInsightsMetricsTagName = MdsdInsightsMetricsSourceName - MdsdKubeMonAgentEventsTagName = MdsdKubeMonAgentEventsSourceName + MdsdKubeMonAgentEventsTagName = MdsdKubeMonAgentEventsSourceName Log("ContainerLogsRouteADX: %v, IsWindows: %v, IsAADMSIAuthMode = %v \n", ContainerLogsRouteADX, IsWindows, IsAADMSIAuthMode) if !ContainerLogsRouteADX && IsWindows && IsAADMSIAuthMode { Log("defaultIngestionAuthTokenRefreshIntervalSeconds = %d \n", defaultIngestionAuthTokenRefreshIntervalSeconds) - IngestionAuthTokenRefreshTicker = time.NewTicker(time.Second * time.Duration(defaultIngestionAuthTokenRefreshIntervalSeconds)) + IngestionAuthTokenRefreshTicker = time.NewTicker(time.Second * time.Duration(defaultIngestionAuthTokenRefreshIntervalSeconds)) go refreshIngestionAuthToken() } } diff --git a/source/plugins/go/src/utils.go b/source/plugins/go/src/utils.go index b0a8d0ba6..02d30607e 100644 --- a/source/plugins/go/src/utils.go +++ b/source/plugins/go/src/utils.go @@ -12,8 +12,8 @@ import ( "net/url" "os" "strings" - "time" - + "time" + "github.com/Azure/azure-kusto-go/kusto" "github.com/Azure/azure-kusto-go/kusto/ingest" "github.com/Azure/go-autorest/autorest/azure/auth" @@ -87,10 +87,9 @@ func CreateHTTPClient() { } tlsConfig.BuildNameToCertificate() - transport = &http.Transport{TLSClientConfig: tlsConfig} + transport = &http.Transport{TLSClientConfig: tlsConfig} } // set the proxy if the proxy configured - // TODO: read proxy info from secret if ProxyEndpoint != "" { proxyEndpointUrl, err := url.Parse(ProxyEndpoint) if err != nil { @@ -106,7 +105,7 @@ func CreateHTTPClient() { HTTPClient = http.Client{ Transport: transport, Timeout: 30 * time.Second, - } + } Log("Successfully created HTTP Client") } @@ -124,57 +123,57 @@ func ToString(s interface{}) string { //mdsdSocketClient to write msgp messages func CreateMDSDClient(dataType DataType, containerType string) { - mdsdfluentSocket := "/var/run/mdsd/default_fluent.socket" + mdsdfluentSocket := "/var/run/mdsd/default_fluent.socket" if containerType != "" && strings.Compare(strings.ToLower(containerType), "prometheussidecar") == 0 { - mdsdfluentSocket = fmt.Sprintf("/var/run/mdsd-%s/default_fluent.socket", containerType) - } + mdsdfluentSocket = fmt.Sprintf("/var/run/mdsd-%s/default_fluent.socket", containerType) + } switch dataType { - case ContainerLogV2: - if MdsdMsgpUnixSocketClient != nil { - MdsdMsgpUnixSocketClient.Close() - MdsdMsgpUnixSocketClient = nil - } - /*conn, err := fluent.New(fluent.Config{FluentNetwork:"unix", - FluentSocketPath:"/var/run/mdsd/default_fluent.socket", - WriteTimeout: 5 * time.Second, - RequestAck: true}) */ - conn, err := net.DialTimeout("unix", - mdsdfluentSocket, 10*time.Second) - if err != nil { - Log("Error::mdsd::Unable to open MDSD msgp socket connection for ContainerLogV2 %s", err.Error()) - //log.Fatalf("Unable to open MDSD msgp socket connection %s", err.Error()) - } else { - Log("Successfully created MDSD msgp socket connection for ContainerLogV2: %s", mdsdfluentSocket) - MdsdMsgpUnixSocketClient = conn - } - case KubeMonAgentEvents: - if MdsdKubeMonMsgpUnixSocketClient != nil { - MdsdKubeMonMsgpUnixSocketClient.Close() - MdsdKubeMonMsgpUnixSocketClient = nil - } - conn, err := net.DialTimeout("unix", - mdsdfluentSocket, 10*time.Second) - if err != nil { - Log("Error::mdsd::Unable to open MDSD msgp socket connection for KubeMon events %s", err.Error()) - //log.Fatalf("Unable to open MDSD msgp socket connection %s", err.Error()) - } else { - Log("Successfully created MDSD msgp socket connection for KubeMon events:%s", mdsdfluentSocket) - MdsdKubeMonMsgpUnixSocketClient = conn - } - case InsightsMetrics: - if MdsdInsightsMetricsMsgpUnixSocketClient != nil { - MdsdInsightsMetricsMsgpUnixSocketClient.Close() - MdsdInsightsMetricsMsgpUnixSocketClient = nil - } - conn, err := net.DialTimeout("unix", - mdsdfluentSocket, 10*time.Second) - if err != nil { - Log("Error::mdsd::Unable to open MDSD msgp socket connection for insights metrics %s", err.Error()) - //log.Fatalf("Unable to open MDSD msgp socket connection %s", err.Error()) - } else { - Log("Successfully created MDSD msgp socket connection for Insights metrics %s", mdsdfluentSocket) - MdsdInsightsMetricsMsgpUnixSocketClient = conn - } + case ContainerLogV2: + if MdsdMsgpUnixSocketClient != nil { + MdsdMsgpUnixSocketClient.Close() + MdsdMsgpUnixSocketClient = nil + } + /*conn, err := fluent.New(fluent.Config{FluentNetwork:"unix", + FluentSocketPath:"/var/run/mdsd/default_fluent.socket", + WriteTimeout: 5 * time.Second, + RequestAck: true}) */ + conn, err := net.DialTimeout("unix", + mdsdfluentSocket, 10*time.Second) + if err != nil { + Log("Error::mdsd::Unable to open MDSD msgp socket connection for ContainerLogV2 %s", err.Error()) + //log.Fatalf("Unable to open MDSD msgp socket connection %s", err.Error()) + } else { + Log("Successfully created MDSD msgp socket connection for ContainerLogV2: %s", mdsdfluentSocket) + MdsdMsgpUnixSocketClient = conn + } + case KubeMonAgentEvents: + if MdsdKubeMonMsgpUnixSocketClient != nil { + MdsdKubeMonMsgpUnixSocketClient.Close() + MdsdKubeMonMsgpUnixSocketClient = nil + } + conn, err := net.DialTimeout("unix", + mdsdfluentSocket, 10*time.Second) + if err != nil { + Log("Error::mdsd::Unable to open MDSD msgp socket connection for KubeMon events %s", err.Error()) + //log.Fatalf("Unable to open MDSD msgp socket connection %s", err.Error()) + } else { + Log("Successfully created MDSD msgp socket connection for KubeMon events:%s", mdsdfluentSocket) + MdsdKubeMonMsgpUnixSocketClient = conn + } + case InsightsMetrics: + if MdsdInsightsMetricsMsgpUnixSocketClient != nil { + MdsdInsightsMetricsMsgpUnixSocketClient.Close() + MdsdInsightsMetricsMsgpUnixSocketClient = nil + } + conn, err := net.DialTimeout("unix", + mdsdfluentSocket, 10*time.Second) + if err != nil { + Log("Error::mdsd::Unable to open MDSD msgp socket connection for insights metrics %s", err.Error()) + //log.Fatalf("Unable to open MDSD msgp socket connection %s", err.Error()) + } else { + Log("Successfully created MDSD msgp socket connection for Insights metrics %s", mdsdfluentSocket) + MdsdInsightsMetricsMsgpUnixSocketClient = conn + } } } @@ -229,7 +228,7 @@ func isValidUrl(uri string) bool { func convertMsgPackEntriesToMsgpBytes(fluentForwardTag string, msgPackEntries []MsgPackEntry) []byte { var msgpBytes []byte - + fluentForward := MsgPackForward{ Tag: fluentForwardTag, Entries: msgPackEntries, @@ -240,7 +239,7 @@ func convertMsgPackEntriesToMsgpBytes(fluentForwardTag string, msgPackEntries [] msgpSize += 1 + msgp.Int64Size + msgp.GuessSize(fluentForward.Entries[i].Record) } - //allocate buffer for msgp message + //allocate buffer for msgp message msgpBytes = msgp.Require(nil, msgpSize) //construct the stream @@ -253,6 +252,6 @@ func convertMsgPackEntriesToMsgpBytes(fluentForwardTag string, msgPackEntries [] msgpBytes = msgp.AppendInt64(msgpBytes, batchTime) msgpBytes = msgp.AppendMapStrStr(msgpBytes, fluentForward.Entries[entry].Record) } - - return msgpBytes + + return msgpBytes } From 16cba94ca3a6fda53af1dd0914d63a22eb4c31ab Mon Sep 17 00:00:00 2001 From: David Michelman Date: Fri, 27 Aug 2021 17:57:24 -0700 Subject: [PATCH 07/14] proxy should be working, but most tables don't have any data. About to merge, maybe whatever was wrong is now fixed --- kubernetes/linux/main.sh | 3 ++- kubernetes/linux/setup.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index bf6073928..f05e548ea 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -156,13 +156,14 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then echo $pwd > /opt/proxy_password echo "export MDSD_PROXY_MODE=application" >> ~/.bashrc - echo "export MDSD_PROXY_ADDRESS=$hostport" >> ~/.bashrc + echo "export MDSD_PROXY_ADDRESS=$proto$hostport" >> ~/.bashrc echo "export MDSD_PROXY_USERNAME=$user" >> ~/.bashrc echo "export MDSD_PROXY_PASSWORD_FILE=/opt/proxy_password" >> ~/.bashrc # TODO: set proxy in go. It's possible that setting $PROXY_ENDPOINT is good enough, but double check # go and ruby should automatically use this env variable echo "export http_proxy=$PROXY_ENDPOINT" >> ~/.bashrc + echo "export https_proxy=$PROXY_ENDPOINT" >> ~/.bashrc source ~/.bashrc fi diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index bac62e7b7..c6d07e62e 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -11,7 +11,7 @@ sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ #install oneagent - Official bits (06/24/2021) # wget https://github.com/microsoft/Docker-Provider/releases/download/06242021-oneagent/azure-mdsd_1.10.3-build.master.241_x86_64.deb -wget https://github.com/microsoft/Docker-Provider/releases/download/0.1/azure-mdsd_1.11.0-build.develop.2091_x86_64.deb # has proxy support +wget https://github.com/microsoft/Docker-Provider/releases/download/oneagent-beta/azure-mdsd_1.13.0-build.develop.2282_x86_64.deb # has proxy support? /usr/bin/dpkg -i $TMPDIR/azure-mdsd*.deb cp -f $TMPDIR/mdsd.xml /etc/mdsd.d From fd5b5d73e0fe2cc7ed845d111afdf8ef69bc848f Mon Sep 17 00:00:00 2001 From: David Michelman Date: Mon, 30 Aug 2021 17:55:10 -0700 Subject: [PATCH 08/14] linux AMA proxy works --- kubernetes/linux/main.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index b31b7ad2d..e8aee836b 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -197,13 +197,15 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then echo "successfully validated provided proxy endpoint is valid and expected format" fi - # Write environment variables so mdsd can use the proxy. Do this regardless of if using mdsd or not because nothing will break because these environment variables are written echo $pwd > /opt/proxy_password echo "export MDSD_PROXY_MODE=application" >> ~/.bashrc echo "export MDSD_PROXY_ADDRESS=$proto$hostport" >> ~/.bashrc echo "export MDSD_PROXY_USERNAME=$user" >> ~/.bashrc echo "export MDSD_PROXY_PASSWORD_FILE=/opt/proxy_password" >> ~/.bashrc + + #TODO: Compression + proxy creates a deserialization error in ODS. Not sure why + echo "export MDSD_ODS_COMPRESSION_LEVEL=0" >> ~/.bashrc # TODO: set proxy in go. It's possible that setting $PROXY_ENDPOINT is good enough, but double check # go and ruby should automatically use this env variable From e09ffabb768a0e0e285c6f5ed81cf0ee4f58ad90 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Fri, 1 Oct 2021 15:35:21 -0700 Subject: [PATCH 09/14] about to merge --- kubernetes/linux/Dockerfile | 2 +- kubernetes/linux/main.sh | 33 ++++++++++++++++++--------------- kubernetes/linux/setup.sh | 2 +- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/kubernetes/linux/Dockerfile b/kubernetes/linux/Dockerfile index 07af7f4a7..fcfa8ac31 100644 --- a/kubernetes/linux/Dockerfile +++ b/kubernetes/linux/Dockerfile @@ -16,7 +16,7 @@ ENV AZMON_COLLECT_ENV False ENV KUBE_CLIENT_BACKOFF_BASE 1 ENV KUBE_CLIENT_BACKOFF_DURATION 0 ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9 -RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg && rm -rf /var/lib/apt/lists/* +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg less nano && rm -rf /var/lib/apt/lists/* COPY setup.sh main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar mdsd.xml envmdsd logrotate.conf $tmpdir/ WORKDIR ${tmpdir} diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index e8aee836b..db905b141 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -101,11 +101,11 @@ if [ -z $AKS_RESOURCE_ID ]; then else export customResourceId=$AKS_RESOURCE_ID echo "export customResourceId=$AKS_RESOURCE_ID" >> ~/.bashrc - source ~/.bashrc + # source ~/.bashrc echo "customResourceId:$customResourceId" export customRegion=$AKS_REGION echo "export customRegion=$AKS_REGION" >> ~/.bashrc - source ~/.bashrc + # source ~/.bashrc echo "customRegion:$customRegion" fi @@ -120,7 +120,7 @@ if [ -e "/etc/config/settings/schema-version" ] && [ -s "/etc/config/settings/ export AZMON_AGENT_CFG_SCHEMA_VERSION=$config_schema_version echo "export AZMON_AGENT_CFG_SCHEMA_VERSION=$config_schema_version" >> ~/.bashrc - source ~/.bashrc + # source ~/.bashrc echo "AZMON_AGENT_CFG_SCHEMA_VERSION:$AZMON_AGENT_CFG_SCHEMA_VERSION" fi @@ -135,7 +135,7 @@ if [ -e "/etc/config/settings/config-version" ] && [ -s "/etc/config/settings/ export AZMON_AGENT_CFG_FILE_VERSION=$config_file_version echo "export AZMON_AGENT_CFG_FILE_VERSION=$config_file_version" >> ~/.bashrc - source ~/.bashrc + # source ~/.bashrc echo "AZMON_AGENT_CFG_FILE_VERSION:$AZMON_AGENT_CFG_FILE_VERSION" fi @@ -152,7 +152,7 @@ if [[ ( ( ! -e "/etc/config/kube.conf" ) && ( "${CONTAINER_TYPE}" == "Prometheus export AZMON_OSM_CFG_SCHEMA_VERSION=$osm_config_schema_version echo "export AZMON_OSM_CFG_SCHEMA_VERSION=$osm_config_schema_version" >> ~/.bashrc - source ~/.bashrc + # source ~/.bashrc echo "AZMON_OSM_CFG_SCHEMA_VERSION:$AZMON_OSM_CFG_SCHEMA_VERSION" fi fi @@ -199,20 +199,23 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then echo $pwd > /opt/proxy_password + export MDSD_PROXY_MODE=application echo "export MDSD_PROXY_MODE=application" >> ~/.bashrc + export MDSD_PROXY_ADDRESS=$proto$hostport echo "export MDSD_PROXY_ADDRESS=$proto$hostport" >> ~/.bashrc + export MDSD_PROXY_USERNAME=$user echo "export MDSD_PROXY_USERNAME=$user" >> ~/.bashrc + export MDSD_PROXY_PASSWORD_FILE=/opt/proxy_password echo "export MDSD_PROXY_PASSWORD_FILE=/opt/proxy_password" >> ~/.bashrc #TODO: Compression + proxy creates a deserialization error in ODS. Not sure why + export MDSD_ODS_COMPRESSION_LEVEL=0 echo "export MDSD_ODS_COMPRESSION_LEVEL=0" >> ~/.bashrc - # TODO: set proxy in go. It's possible that setting $PROXY_ENDPOINT is good enough, but double check - # go and ruby should automatically use this env variable - echo "export http_proxy=$PROXY_ENDPOINT" >> ~/.bashrc - echo "export https_proxy=$PROXY_ENDPOINT" >> ~/.bashrc - - source ~/.bashrc + # # TODO: set proxy in go. It's possible that setting $PROXY_ENDPOINT is good enough, but double check + # # go and ruby should automatically use this env variable + # echo "export http_proxy=$PROXY_ENDPOINT" >> ~/.bashrc + # echo "export https_proxy=$PROXY_ENDPOINT" >> ~/.bashrc fi if [ ! -z "$PROXY_ENDPOINT" ]; then @@ -310,7 +313,7 @@ aikey=$(echo $APPLICATIONINSIGHTS_AUTH | base64 --decode) export TELEMETRY_APPLICATIONINSIGHTS_KEY=$aikey echo "export TELEMETRY_APPLICATIONINSIGHTS_KEY=$aikey" >> ~/.bashrc -source ~/.bashrc +# source ~/.bashrc if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then #Parse the configmap to set the right environment variables. @@ -502,7 +505,7 @@ sudo setcap cap_sys_ptrace,cap_dac_read_search+ep /usr/bin/ruby2.6 echo "export KUBELET_RUNTIME_OPERATIONS_METRIC="$KUBELET_RUNTIME_OPERATIONS_METRIC >> ~/.bashrc echo "export KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC="$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC >> ~/.bashrc -source ~/.bashrc +# source ~/.bashrc echo $NODE_NAME > /var/opt/microsoft/docker-cimprov/state/containerhostname #check if file was written successfully. @@ -562,7 +565,7 @@ else export MDSD_FLUENT_SOCKET_PORT="29230" echo "export MDSD_FLUENT_SOCKET_PORT=$MDSD_FLUENT_SOCKET_PORT" >> ~/.bashrc fi -source ~/.bashrc +# source ~/.bashrc dpkg -l | grep mdsd | awk '{print $2 " " $3}' @@ -574,7 +577,7 @@ if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then echo "export TENANT_NAME=$TENANT_NAME" >> ~/.bashrc export MDSD_ROLE_PREFIX=/var/run/mdsd-${CONTAINER_TYPE}/default echo "export MDSD_ROLE_PREFIX=$MDSD_ROLE_PREFIX" >> ~/.bashrc - source ~/.bashrc + # source ~/.bashrc mkdir /var/run/mdsd-${CONTAINER_TYPE} # add -T 0xFFFF for full traces mdsd ${MDSD_AAD_MSI_AUTH_ARGS} -r ${MDSD_ROLE_PREFIX} -p 26130 -f 26230 -i 26330 -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos & diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 249b75883..ad4ea7719 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -11,7 +11,7 @@ sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ #install oneagent - Official bits (06/24/2021) # wget https://github.com/microsoft/Docker-Provider/releases/download/06242021-oneagent/azure-mdsd_1.10.3-build.master.241_x86_64.deb -wget https://github.com/microsoft/Docker-Provider/releases/download/oneagent-beta/azure-mdsd_1.13.0-build.develop.2282_x86_64.deb # has proxy support? +wget https://github.com/microsoft/Docker-Provider/releases/download/oneagent-beta/azure-mdsd_1.13.0-build.develop.2294_x86_64.deb # has proxy support? /usr/bin/dpkg -i $TMPDIR/azure-mdsd*.deb cp -f $TMPDIR/mdsd.xml /etc/mdsd.d From c610b8c2234e64b2cf716c9ad5d041a5211035a9 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Thu, 7 Oct 2021 15:16:40 -0700 Subject: [PATCH 10/14] proxy support appears to be working, final mdsd build location will still change --- kubernetes/linux/main.sh | 5 ----- kubernetes/linux/setup.sh | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index db905b141..b9be2cbe8 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -211,11 +211,6 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then #TODO: Compression + proxy creates a deserialization error in ODS. Not sure why export MDSD_ODS_COMPRESSION_LEVEL=0 echo "export MDSD_ODS_COMPRESSION_LEVEL=0" >> ~/.bashrc - - # # TODO: set proxy in go. It's possible that setting $PROXY_ENDPOINT is good enough, but double check - # # go and ruby should automatically use this env variable - # echo "export http_proxy=$PROXY_ENDPOINT" >> ~/.bashrc - # echo "export https_proxy=$PROXY_ENDPOINT" >> ~/.bashrc fi if [ ! -z "$PROXY_ENDPOINT" ]; then diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index ad4ea7719..c5b690077 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -11,7 +11,7 @@ sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ #install oneagent - Official bits (06/24/2021) # wget https://github.com/microsoft/Docker-Provider/releases/download/06242021-oneagent/azure-mdsd_1.10.3-build.master.241_x86_64.deb -wget https://github.com/microsoft/Docker-Provider/releases/download/oneagent-beta/azure-mdsd_1.13.0-build.develop.2294_x86_64.deb # has proxy support? +wget https://github.com/microsoft/Docker-Provider/releases/download/oneagent-beta/azure-mdsd_1.12.3-build.develop.275_x86_64.deb # has proxy support? /usr/bin/dpkg -i $TMPDIR/azure-mdsd*.deb cp -f $TMPDIR/mdsd.xml /etc/mdsd.d From 9682796d56f95773c93e8979996b6a3bf3df490e Mon Sep 17 00:00:00 2001 From: David Michelman Date: Thu, 7 Oct 2021 15:59:25 -0700 Subject: [PATCH 11/14] removing some unnecessary changes --- kubernetes/linux/Dockerfile | 2 +- kubernetes/linux/main.sh | 20 ++++++++++---------- kubernetes/linux/setup.sh | 5 ++--- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/kubernetes/linux/Dockerfile b/kubernetes/linux/Dockerfile index fcfa8ac31..07af7f4a7 100644 --- a/kubernetes/linux/Dockerfile +++ b/kubernetes/linux/Dockerfile @@ -16,7 +16,7 @@ ENV AZMON_COLLECT_ENV False ENV KUBE_CLIENT_BACKOFF_BASE 1 ENV KUBE_CLIENT_BACKOFF_DURATION 0 ENV RUBY_GC_HEAP_OLDOBJECT_LIMIT_FACTOR 0.9 -RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg less nano && rm -rf /var/lib/apt/lists/* +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -y libc-bin wget openssl curl sudo python-ctypes init-system-helpers net-tools rsyslog cron vim dmidecode apt-transport-https gnupg && rm -rf /var/lib/apt/lists/* COPY setup.sh main.sh defaultpromenvvariables defaultpromenvvariables-rs defaultpromenvvariables-sidecar mdsd.xml envmdsd logrotate.conf $tmpdir/ WORKDIR ${tmpdir} diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index b9be2cbe8..18366aa5c 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -101,11 +101,11 @@ if [ -z $AKS_RESOURCE_ID ]; then else export customResourceId=$AKS_RESOURCE_ID echo "export customResourceId=$AKS_RESOURCE_ID" >> ~/.bashrc - # source ~/.bashrc + source ~/.bashrc echo "customResourceId:$customResourceId" export customRegion=$AKS_REGION echo "export customRegion=$AKS_REGION" >> ~/.bashrc - # source ~/.bashrc + source ~/.bashrc echo "customRegion:$customRegion" fi @@ -120,7 +120,7 @@ if [ -e "/etc/config/settings/schema-version" ] && [ -s "/etc/config/settings/ export AZMON_AGENT_CFG_SCHEMA_VERSION=$config_schema_version echo "export AZMON_AGENT_CFG_SCHEMA_VERSION=$config_schema_version" >> ~/.bashrc - # source ~/.bashrc + source ~/.bashrc echo "AZMON_AGENT_CFG_SCHEMA_VERSION:$AZMON_AGENT_CFG_SCHEMA_VERSION" fi @@ -135,7 +135,7 @@ if [ -e "/etc/config/settings/config-version" ] && [ -s "/etc/config/settings/ export AZMON_AGENT_CFG_FILE_VERSION=$config_file_version echo "export AZMON_AGENT_CFG_FILE_VERSION=$config_file_version" >> ~/.bashrc - # source ~/.bashrc + source ~/.bashrc echo "AZMON_AGENT_CFG_FILE_VERSION:$AZMON_AGENT_CFG_FILE_VERSION" fi @@ -152,7 +152,7 @@ if [[ ( ( ! -e "/etc/config/kube.conf" ) && ( "${CONTAINER_TYPE}" == "Prometheus export AZMON_OSM_CFG_SCHEMA_VERSION=$osm_config_schema_version echo "export AZMON_OSM_CFG_SCHEMA_VERSION=$osm_config_schema_version" >> ~/.bashrc - # source ~/.bashrc + source ~/.bashrc echo "AZMON_OSM_CFG_SCHEMA_VERSION:$AZMON_OSM_CFG_SCHEMA_VERSION" fi fi @@ -208,7 +208,7 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then export MDSD_PROXY_PASSWORD_FILE=/opt/proxy_password echo "export MDSD_PROXY_PASSWORD_FILE=/opt/proxy_password" >> ~/.bashrc - #TODO: Compression + proxy creates a deserialization error in ODS. Not sure why + #TODO: Compression + proxy creates a deserialization error in ODS. This needs a fix in MDSD export MDSD_ODS_COMPRESSION_LEVEL=0 echo "export MDSD_ODS_COMPRESSION_LEVEL=0" >> ~/.bashrc fi @@ -308,7 +308,7 @@ aikey=$(echo $APPLICATIONINSIGHTS_AUTH | base64 --decode) export TELEMETRY_APPLICATIONINSIGHTS_KEY=$aikey echo "export TELEMETRY_APPLICATIONINSIGHTS_KEY=$aikey" >> ~/.bashrc -# source ~/.bashrc +source ~/.bashrc if [ "${CONTAINER_TYPE}" != "PrometheusSidecar" ]; then #Parse the configmap to set the right environment variables. @@ -500,7 +500,7 @@ sudo setcap cap_sys_ptrace,cap_dac_read_search+ep /usr/bin/ruby2.6 echo "export KUBELET_RUNTIME_OPERATIONS_METRIC="$KUBELET_RUNTIME_OPERATIONS_METRIC >> ~/.bashrc echo "export KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC="$KUBELET_RUNTIME_OPERATIONS_ERRORS_METRIC >> ~/.bashrc -# source ~/.bashrc +source ~/.bashrc echo $NODE_NAME > /var/opt/microsoft/docker-cimprov/state/containerhostname #check if file was written successfully. @@ -560,7 +560,7 @@ else export MDSD_FLUENT_SOCKET_PORT="29230" echo "export MDSD_FLUENT_SOCKET_PORT=$MDSD_FLUENT_SOCKET_PORT" >> ~/.bashrc fi -# source ~/.bashrc +source ~/.bashrc dpkg -l | grep mdsd | awk '{print $2 " " $3}' @@ -572,7 +572,7 @@ if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then echo "export TENANT_NAME=$TENANT_NAME" >> ~/.bashrc export MDSD_ROLE_PREFIX=/var/run/mdsd-${CONTAINER_TYPE}/default echo "export MDSD_ROLE_PREFIX=$MDSD_ROLE_PREFIX" >> ~/.bashrc - # source ~/.bashrc + source ~/.bashrc mkdir /var/run/mdsd-${CONTAINER_TYPE} # add -T 0xFFFF for full traces mdsd ${MDSD_AAD_MSI_AUTH_ARGS} -r ${MDSD_ROLE_PREFIX} -p 26130 -f 26230 -i 26330 -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos & diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index c5b690077..4459935ac 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -9,9 +9,8 @@ sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ dpkg-reconfigure --frontend=noninteractive locales && \ update-locale LANG=en_US.UTF-8 -#install oneagent - Official bits (06/24/2021) -# wget https://github.com/microsoft/Docker-Provider/releases/download/06242021-oneagent/azure-mdsd_1.10.3-build.master.241_x86_64.deb -wget https://github.com/microsoft/Docker-Provider/releases/download/oneagent-beta/azure-mdsd_1.12.3-build.develop.275_x86_64.deb # has proxy support? +#install oneagent - Official bits (10/7/2021) +wget https://github.com/microsoft/Docker-Provider/releases/download/oneagent-beta/azure-mdsd_1.12.3-build.develop.275_x86_64.deb /usr/bin/dpkg -i $TMPDIR/azure-mdsd*.deb cp -f $TMPDIR/mdsd.xml /etc/mdsd.d From a4149a07266f2ddfbbf841573db5413381741d0f Mon Sep 17 00:00:00 2001 From: David Michelman Date: Thu, 7 Oct 2021 16:00:10 -0700 Subject: [PATCH 12/14] forgot to remove one last change --- kubernetes/linux/main.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index 18366aa5c..8a4b817bd 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -158,7 +158,6 @@ if [[ ( ( ! -e "/etc/config/kube.conf" ) && ( "${CONTAINER_TYPE}" == "Prometheus fi export PROXY_ENDPOINT="" -# TODO: here # Check for internet connectivity or workspace deletion if [ -e "/etc/omsagent-secret/WSID" ]; then workspaceId=$(cat /etc/omsagent-secret/WSID) From 6760f2778c080777a50b2d70df3e6dc7ad3f9ba0 Mon Sep 17 00:00:00 2001 From: David Michelman Date: Thu, 7 Oct 2021 18:10:02 -0700 Subject: [PATCH 13/14] redirected mdsd stderr to stdout instead of stdin --- kubernetes/linux/main.sh | 12 ++++++------ kubernetes/linux/setup.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index 8a4b817bd..2a95af5e2 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -199,17 +199,17 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then echo $pwd > /opt/proxy_password export MDSD_PROXY_MODE=application - echo "export MDSD_PROXY_MODE=application" >> ~/.bashrc + echo "export MDSD_PROXY_MODE=$MDSD_PROXY_MODE" >> ~/.bashrc export MDSD_PROXY_ADDRESS=$proto$hostport - echo "export MDSD_PROXY_ADDRESS=$proto$hostport" >> ~/.bashrc + echo "export MDSD_PROXY_ADDRESS=$MDSD_PROXY_ADDRESS" >> ~/.bashrc export MDSD_PROXY_USERNAME=$user - echo "export MDSD_PROXY_USERNAME=$user" >> ~/.bashrc + echo "export MDSD_PROXY_USERNAME=$MDSD_PROXY_USERNAME" >> ~/.bashrc export MDSD_PROXY_PASSWORD_FILE=/opt/proxy_password - echo "export MDSD_PROXY_PASSWORD_FILE=/opt/proxy_password" >> ~/.bashrc + echo "export MDSD_PROXY_PASSWORD_FILE=$MDSD_PROXY_PASSWORD_FILE" >> ~/.bashrc #TODO: Compression + proxy creates a deserialization error in ODS. This needs a fix in MDSD export MDSD_ODS_COMPRESSION_LEVEL=0 - echo "export MDSD_ODS_COMPRESSION_LEVEL=0" >> ~/.bashrc + echo "export MDSD_ODS_COMPRESSION_LEVEL=$MDSD_ODS_COMPRESSION_LEVEL" >> ~/.bashrc fi if [ ! -z "$PROXY_ENDPOINT" ]; then @@ -578,7 +578,7 @@ if [ "${CONTAINER_TYPE}" == "PrometheusSidecar" ]; then else echo "starting mdsd mode in main container..." # add -T 0xFFFF for full traces - mdsd ${MDSD_AAD_MSI_AUTH_ARGS} -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos & + mdsd ${MDSD_AAD_MSI_AUTH_ARGS} -e ${MDSD_LOG}/mdsd.err -w ${MDSD_LOG}/mdsd.warn -o ${MDSD_LOG}/mdsd.info -q ${MDSD_LOG}/mdsd.qos 2>> /dev/null & fi # Set up a cron job for logrotation diff --git a/kubernetes/linux/setup.sh b/kubernetes/linux/setup.sh index 4459935ac..371d26fa5 100644 --- a/kubernetes/linux/setup.sh +++ b/kubernetes/linux/setup.sh @@ -10,7 +10,7 @@ sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \ update-locale LANG=en_US.UTF-8 #install oneagent - Official bits (10/7/2021) -wget https://github.com/microsoft/Docker-Provider/releases/download/oneagent-beta/azure-mdsd_1.12.3-build.develop.275_x86_64.deb +wget https://github.com/microsoft/Docker-Provider/releases/download/1.14/azure-mdsd_1.14.0-build.master.279_x86_64.deb /usr/bin/dpkg -i $TMPDIR/azure-mdsd*.deb cp -f $TMPDIR/mdsd.xml /etc/mdsd.d From e4538542cd4ca3c3a1d2a6de68e78f5cddc735cb Mon Sep 17 00:00:00 2001 From: David Michelman Date: Thu, 7 Oct 2021 18:13:36 -0700 Subject: [PATCH 14/14] addressing proxy password location comment --- kubernetes/linux/main.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/linux/main.sh b/kubernetes/linux/main.sh index 2a95af5e2..a9184ab53 100644 --- a/kubernetes/linux/main.sh +++ b/kubernetes/linux/main.sh @@ -196,7 +196,7 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then echo "successfully validated provided proxy endpoint is valid and expected format" fi - echo $pwd > /opt/proxy_password + echo $pwd > /opt/microsoft/docker-cimprov/proxy_password export MDSD_PROXY_MODE=application echo "export MDSD_PROXY_MODE=$MDSD_PROXY_MODE" >> ~/.bashrc @@ -204,7 +204,7 @@ if [ -e "/etc/omsagent-secret/WSID" ]; then echo "export MDSD_PROXY_ADDRESS=$MDSD_PROXY_ADDRESS" >> ~/.bashrc export MDSD_PROXY_USERNAME=$user echo "export MDSD_PROXY_USERNAME=$MDSD_PROXY_USERNAME" >> ~/.bashrc - export MDSD_PROXY_PASSWORD_FILE=/opt/proxy_password + export MDSD_PROXY_PASSWORD_FILE=/opt/microsoft/docker-cimprov/proxy_password echo "export MDSD_PROXY_PASSWORD_FILE=$MDSD_PROXY_PASSWORD_FILE" >> ~/.bashrc #TODO: Compression + proxy creates a deserialization error in ODS. This needs a fix in MDSD