From d9dbbd8ebb9cfac8df066b972e86f68c28a8ba3d Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 18 Oct 2018 17:48:23 -0700 Subject: [PATCH 01/45] first stab --- source/code/plugin/DockerApiClient.rb | 0 source/code/plugin/socketClient.rb | 85 +++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 source/code/plugin/DockerApiClient.rb create mode 100644 source/code/plugin/socketClient.rb diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb new file mode 100644 index 000000000..e69de29bb diff --git a/source/code/plugin/socketClient.rb b/source/code/plugin/socketClient.rb new file mode 100644 index 000000000..9b76dc790 --- /dev/null +++ b/source/code/plugin/socketClient.rb @@ -0,0 +1,85 @@ +require "socket" +require 'json' + +socket = UNIXSocket.new('/var/run/docker.sock') + +socket.write("GET /info HTTP/1.1\r\nHost: localhost\r\n\r\n") +myresponse = "" +loop do +mystring = socket.recv(4096) +myresponse = myresponse + mystring +#puts "--------------------------------" +break if mystring.length < 4096 +end +puts myresponse +myjson = myresponse.match( /{.+}/ )[0] +myjson = "["+myjson+"]" +myparsedjson = JSON.parse(myjson) +puts "Hostname-" + myparsedjson[0]['Name'] + +#puts "==== Sending" +socket.write("GET /images/json?all=0 HTTP/1.1\r\nHost: localhost\r\n\r\n") + +#socket.write("GET /info HTTP/1.1\r\nHost: localhost\r\n\r\n") + +#socket.write("info HTTP/1.1\r\nHost: localhost\r\n\r\n") + +#puts "==== Getting Response" +#puts socket + +#puts socket.recvfrom(1024) + +#puts socket.gets + +#puts socket.recv.body + +myresponse = "" +loop do +mystring = socket.recv(4096) +myresponse = myresponse + mystring +#puts "--------------------------------" +break if mystring.length < 4096 +end + +#puts myresponse +#puts "-----------------------------------------------------" + +myjson = myresponse.match( /{.+}/ )[0] +myjson = "["+myjson+"]" +#puts myjson + + +myparsedjson = JSON.parse(myjson) +puts"---------------------------" +puts"IMAGES" +puts"---------------------------" +myparsedjson.each do |items| + puts items['Id'] +end +#while socket.recvfrom(1024) +# puts socket.recvfrom(1024) +#end + +#puts "here" +#puts "==== Sending" +socket.write("GET /containers/json?all=1 HTTP/1.1\r\nHost: localhost\r\n\r\n") +#puts "==== Getting Response" + +myresponse = "" +loop do +mystring = socket.recv(4096) +myresponse = myresponse + mystring +#puts "--------------------------------" +break if mystring.length < 4096 +end +myjson = myresponse.match( /{.+}/ )[0] +myjson = "["+myjson+"]" +myparsedjson = JSON.parse(myjson) +#puts myparsedjson +puts"---------------------------" +puts"CONTAINERS" +puts"---------------------------" +myparsedjson.each do |items| + puts items['Id'] +end +socket.close \ No newline at end of file From f7f6e3d0e5f4d9de7cd347572eb81b23993b52c7 Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 22 Oct 2018 14:04:46 -0700 Subject: [PATCH 02/45] changes --- source/code/plugin/DockerApiClient.rb | 18 +++++++++ source/code/plugin/DockerApiRestHelper.rb | 48 +++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 source/code/plugin/DockerApiRestHelper.rb diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index e69de29bb..e6d2ee061 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -0,0 +1,18 @@ +#!/usr/local/bin/ruby +# frozen_string_literal: true + +class DockerApiClient + + require 'socket' + require 'json' + require_relative 'oms_common' + + @@SocketPath = "/var/run/docker.sock" + @@ChunkSize = 4096 + + + + +end + + \ No newline at end of file diff --git a/source/code/plugin/DockerApiRestHelper.rb b/source/code/plugin/DockerApiRestHelper.rb new file mode 100644 index 000000000..9d14a75c6 --- /dev/null +++ b/source/code/plugin/DockerApiRestHelper.rb @@ -0,0 +1,48 @@ +#!/usr/local/bin/ruby +# frozen_string_literal: true + +class DockerApiRestHelper + def initialize + end + + class << self + # Create the REST request to list images + # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.21/#list-images + # returns Request in string format + def restDockerImages() + begin + return "GET /images/json?all=0 HTTP/1.1\r\nHost: localhost\r\n\r\n"; + end + end + + # Create the REST request to list containers + # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.21/#list-containers + # returns Request in string format + def restDockerPs() + begin + return "GET /containers/json?all=1 HTTP/1.1\r\nHost: localhost\r\n\r\n"; + end + end + + # Create the REST request to list running containers + # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.21/#list-containers + # returns Request in string format + def restDockerPsRunning() + begin + return "GET /containers/json HTTP/1.1\r\nHost: localhost\r\n\r\n"; + end + end + + # Create the REST request to inspect a container + # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.21/#inspect-a-container + # parameter - ID of the container to be inspected + # returns Request in string format + def restDockerInspect(id) + begin + return "GET /containers/" + id + "/json HTTP/1.1\r\nHost: localhost\r\n\r\n"; + end + end + +end + + \ No newline at end of file From 5b215bfe371cab3af204173a0ea55eda770ec6c7 Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 22 Oct 2018 16:13:13 -0700 Subject: [PATCH 03/45] changes --- source/code/plugin/DockerApiClient.rb | 46 ++++++++++++++++++++++- source/code/plugin/DockerApiRestHelper.rb | 9 +++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index e6d2ee061..97cfb51b6 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -6,12 +6,56 @@ class DockerApiClient require 'socket' require 'json' require_relative 'oms_common' + require_relative 'DockerApiRestHelper' @@SocketPath = "/var/run/docker.sock" @@ChunkSize = 4096 - + def initialize + end + # Make docker socket call to + def getResponse(request, isMultiJson) + #TODO: Add error handling and retries + socket = UnixSocket.New(@@SocketPath) + dockerResponse = "" + socket.write(request) + # iterate through the response until the last chunk is less than the chunk size so that we can read all data in socket. + loop do + responseChunk = socket.recv(@@ChunkSize) + dockerResponse += responseChunk + break if responseChunk.length < @@ChunkSize + end + socket.close + return parseResponse(dockerResponse, isMultiJson) + end + + def parseResponse(dockerResponse, isMultiJson) + # Doing this because the response is in the raw format and includes headers. + # Need to do a regex match to extract the json part of the response - Anything between [{}] in response + begin + jsonResponse = isMultiJson ? dockerResponse[/\[{.+}\]/] : dockerResponse[/{.+}/] + rescue => exception + @Log.warn("Regex match for docker response failed: #{exception} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") + end + begin + parsedJsonResponse = JSON.parse(jsonResponse) + rescue => exception + @Log.warn("Json parsing for docker response failed: #{exception} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") + end + return parsedJsonResponse + end + + + def getDockerHostName() + dockerHostName = "" + request = DockerApiRestHelper.restDockerInfo() + response = getResponse(request, false) + if (response != nil) + dockerHostName = reponse['Name'] + end + return dockerHostName + end end diff --git a/source/code/plugin/DockerApiRestHelper.rb b/source/code/plugin/DockerApiRestHelper.rb index 9d14a75c6..d528a0290 100644 --- a/source/code/plugin/DockerApiRestHelper.rb +++ b/source/code/plugin/DockerApiRestHelper.rb @@ -43,6 +43,15 @@ def restDockerInspect(id) end end + # Create the REST request to get docker info + # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.21/#get-container-stats-based-on-resource-usage + # returns Request in string format + def restDockerInfo() + begin + return "GET /info HTTP/1.1\r\nHost: localhost\r\n\r\n"; + end + end + end \ No newline at end of file From 524f1bbc478c14047dae19d145474f4b98fe3164 Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 22 Oct 2018 16:26:03 -0700 Subject: [PATCH 04/45] docker util changes --- source/code/plugin/DockerApiClient.rb | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index 97cfb51b6..b5d8be649 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -33,13 +33,16 @@ def getResponse(request, isMultiJson) def parseResponse(dockerResponse, isMultiJson) # Doing this because the response is in the raw format and includes headers. # Need to do a regex match to extract the json part of the response - Anything between [{}] in response + parsedJsonResponse = nil begin jsonResponse = isMultiJson ? dockerResponse[/\[{.+}\]/] : dockerResponse[/{.+}/] rescue => exception @Log.warn("Regex match for docker response failed: #{exception} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") end begin - parsedJsonResponse = JSON.parse(jsonResponse) + if jsonResponse != nil + parsedJsonResponse = JSON.parse(jsonResponse) + end rescue => exception @Log.warn("Json parsing for docker response failed: #{exception} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") end @@ -57,6 +60,16 @@ def getDockerHostName() return dockerHostName end + def listContainers() + ids = [] + request = DockerApiRestHelper.restDockerPs() + containers = getResponse(request, true) + containers.each do |container| + ids.push container['Id'] + end + return ids + end + end \ No newline at end of file From 9b95f457a8fa2b0d06af6163fd6b87ff5fb17f96 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 23 Oct 2018 11:29:14 -0700 Subject: [PATCH 05/45] working tested util --- source/code/plugin/DockerApiClient.rb | 94 +++++++------- source/code/plugin/DockerApiRestHelper.rb | 2 +- source/code/plugin/in_container_inventory.rb | 122 +++++++++++++++++++ source/code/plugin/testdockerclient.rb | 6 + 4 files changed, 177 insertions(+), 47 deletions(-) create mode 100644 source/code/plugin/in_container_inventory.rb create mode 100644 source/code/plugin/testdockerclient.rb diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index b5d8be649..ba191b98d 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -6,6 +6,7 @@ class DockerApiClient require 'socket' require 'json' require_relative 'oms_common' + require_relative 'omslog' require_relative 'DockerApiRestHelper' @@SocketPath = "/var/run/docker.sock" @@ -14,62 +15,63 @@ class DockerApiClient def initialize end - # Make docker socket call to - def getResponse(request, isMultiJson) - #TODO: Add error handling and retries - socket = UnixSocket.New(@@SocketPath) - dockerResponse = "" - socket.write(request) - # iterate through the response until the last chunk is less than the chunk size so that we can read all data in socket. - loop do - responseChunk = socket.recv(@@ChunkSize) - dockerResponse += responseChunk - break if responseChunk.length < @@ChunkSize + class << self + # Make docker socket call to + def getResponse(request, isMultiJson) + #TODO: Add error handling and retries + socket = UNIXSocket.new(@@SocketPath) + dockerResponse = "" + socket.write(request) + # iterate through the response until the last chunk is less than the chunk size so that we can read all data in socket. + loop do + responseChunk = socket.recv(@@ChunkSize) + dockerResponse += responseChunk + break if responseChunk.length < @@ChunkSize + end + socket.close + return parseResponse(dockerResponse, isMultiJson) end - socket.close - return parseResponse(dockerResponse, isMultiJson) - end - def parseResponse(dockerResponse, isMultiJson) - # Doing this because the response is in the raw format and includes headers. - # Need to do a regex match to extract the json part of the response - Anything between [{}] in response - parsedJsonResponse = nil - begin - jsonResponse = isMultiJson ? dockerResponse[/\[{.+}\]/] : dockerResponse[/{.+}/] - rescue => exception - @Log.warn("Regex match for docker response failed: #{exception} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") - end - begin - if jsonResponse != nil - parsedJsonResponse = JSON.parse(jsonResponse) + def parseResponse(dockerResponse, isMultiJson) + # Doing this because the response is in the raw format and includes headers. + # Need to do a regex match to extract the json part of the response - Anything between [{}] in response + parsedJsonResponse = nil + begin + jsonResponse = isMultiJson ? dockerResponse[/\[{.+}\]/] : dockerResponse[/{.+}/] + rescue => exception + @Log.warn("Regex match for docker response failed: #{exception} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") end - rescue => exception - @Log.warn("Json parsing for docker response failed: #{exception} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") + begin + if jsonResponse != nil + parsedJsonResponse = JSON.parse(jsonResponse) + end + rescue => exception + @Log.warn("Json parsing for docker response failed: #{exception} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") + end + return parsedJsonResponse end - return parsedJsonResponse - end - def getDockerHostName() - dockerHostName = "" - request = DockerApiRestHelper.restDockerInfo() - response = getResponse(request, false) - if (response != nil) - dockerHostName = reponse['Name'] + def getDockerHostName() + dockerHostName = "" + request = DockerApiRestHelper.restDockerInfo() + response = getResponse(request, false) + if (response != nil) + dockerHostName = response['Name'] + end + return dockerHostName end - return dockerHostName - end - def listContainers() - ids = [] - request = DockerApiRestHelper.restDockerPs() - containers = getResponse(request, true) - containers.each do |container| - ids.push container['Id'] + def listContainers() + ids = [] + request = DockerApiRestHelper.restDockerPs() + containers = getResponse(request, true) + containers.each do |container| + ids.push container['Id'] + end + return ids end - return ids end - end \ No newline at end of file diff --git a/source/code/plugin/DockerApiRestHelper.rb b/source/code/plugin/DockerApiRestHelper.rb index d528a0290..8a1e1585e 100644 --- a/source/code/plugin/DockerApiRestHelper.rb +++ b/source/code/plugin/DockerApiRestHelper.rb @@ -51,7 +51,7 @@ def restDockerInfo() return "GET /info HTTP/1.1\r\nHost: localhost\r\n\r\n"; end end - + end end \ No newline at end of file diff --git a/source/code/plugin/in_container_inventory.rb b/source/code/plugin/in_container_inventory.rb new file mode 100644 index 000000000..ed07447f9 --- /dev/null +++ b/source/code/plugin/in_container_inventory.rb @@ -0,0 +1,122 @@ +#!/usr/local/bin/ruby +# frozen_string_literal: true + +module Fluent + + class Container_Inventory_Input < Input + Plugin.register_input('containerinventory', self) + + def initialize + super + require 'yaml' + require 'json' + + require_relative 'DockerApiClient' + #require_relative 'oms_common' + #require_relative 'omslog' + end + + config_param :run_interval, :time, :default => '1m' + config_param :tag, :string, :default => "oms.ContainerInventory.CollectionTime" + + def configure (conf) + super + end + + def start + if @run_interval + @finished = false + @condition = ConditionVariable.new + @mutex = Mutex.new + @thread = Thread.new(&method(:run_periodic)) + end + end + + def shutdown + if @run_interval + @mutex.synchronize { + @finished = true + @condition.signal + } + @thread.join + end + end +=begin + def enumerate(eventList = nil) + currentTime = Time.now + emitTime = currentTime.to_f + batchTime = currentTime.utc.iso8601 + if eventList.nil? + $log.info("in_kube_events::enumerate : Getting events from Kube API @ #{Time.now.utc.iso8601}") + events = JSON.parse(KubernetesApiClient.getKubeResourceInfo('events').body) + $log.info("in_kube_events::enumerate : Done getting events from Kube API @ #{Time.now.utc.iso8601}") + else + events = eventList + end + eventQueryState = getEventQueryState + newEventQueryState = [] + begin + if(!events.empty?) + eventStream = MultiEventStream.new + events['items'].each do |items| + record = {} + record['CollectionTime'] = batchTime #This is the time that is mapped to become TimeGenerated + eventId = items['metadata']['uid'] + "/" + items['count'].to_s + newEventQueryState.push(eventId) + if !eventQueryState.empty? && eventQueryState.include?(eventId) + next + end + record['ObjectKind']= items['involvedObject']['kind'] + record['Namespace'] = items['involvedObject']['namespace'] + record['Name'] = items['involvedObject']['name'] + record['Reason'] = items['reason'] + record['Message'] = items['message'] + record['Type'] = items['type'] + record['TimeGenerated'] = items['metadata']['creationTimestamp'] + record['SourceComponent'] = items['source']['component'] + record['FirstSeen'] = items['firstTimestamp'] + record['LastSeen'] = items['lastTimestamp'] + record['Count'] = items['count'] + if items['source'].key?('host') + record['Computer'] = items['source']['host'] + else + record['Computer'] = (OMS::Common.get_hostname) + end + record['ClusterName'] = KubernetesApiClient.getClusterName + record['ClusterId'] = KubernetesApiClient.getClusterId + eventStream.add(emitTime, record) if record + end + router.emit_stream(@tag, eventStream) if eventStream + end + writeEventQueryState(newEventQueryState) + rescue => errorStr + $log.warn line.dump, error: errorStr.to_s + $log.debug_backtrace(errorStr.backtrace) + end + end +=end + def run_periodic + @mutex.lock + done = @finished + until done + @condition.wait(@mutex, @run_interval) + done = @finished + @mutex.unlock + if !done + begin + $log.info("in_kube_events::run_periodic @ #{Time.now.utc.iso8601}") + myhost = DockerApiClient.getDockerHostName() + router.emit_stream(@tag, myhost) if myhost + rescue => errorStr + $log.warn "in_kube_events::run_periodic: enumerate Failed to retrieve kube events: #{errorStr}" + end + end + @mutex.lock + end + @mutex.unlock + end + + end # Kube_Event_Input + +end # module + diff --git a/source/code/plugin/testdockerclient.rb b/source/code/plugin/testdockerclient.rb new file mode 100644 index 000000000..cadcad44d --- /dev/null +++ b/source/code/plugin/testdockerclient.rb @@ -0,0 +1,6 @@ +require_relative 'DockerApiClient' + +myhost = DockerApiClient.getDockerHostName +puts myhost + +puts DockerApiClient.listContainers \ No newline at end of file From b0467cfdaf06d4c0a878d1c3154d270224cedabd Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 23 Oct 2018 11:55:01 -0700 Subject: [PATCH 06/45] input plugin and conf --- installer/conf/container.conf | 11 +++ source/code/plugin/in_container_inventory.rb | 87 +++++--------------- 2 files changed, 32 insertions(+), 66 deletions(-) diff --git a/installer/conf/container.conf b/installer/conf/container.conf index 17317871c..bb341e33b 100755 --- a/installer/conf/container.conf +++ b/installer/conf/container.conf @@ -17,6 +17,13 @@ ] + + type containerinventory + tag oms.containerinsights.ContainerInventory + run_interval 60s + log_level debug + + # Image inventory type omi @@ -50,6 +57,10 @@ type filter_container + + @type stdout + + type out_oms_api log_level debug diff --git a/source/code/plugin/in_container_inventory.rb b/source/code/plugin/in_container_inventory.rb index ed07447f9..fe8f9be6d 100644 --- a/source/code/plugin/in_container_inventory.rb +++ b/source/code/plugin/in_container_inventory.rb @@ -8,17 +8,13 @@ class Container_Inventory_Input < Input def initialize super - require 'yaml' require 'json' - require_relative 'DockerApiClient' - #require_relative 'oms_common' - #require_relative 'omslog' end config_param :run_interval, :time, :default => '1m' - config_param :tag, :string, :default => "oms.ContainerInventory.CollectionTime" - + config_param :tag, :string, :default => "oms.containerinsights.ContainerInventory" + def configure (conf) super end @@ -41,60 +37,21 @@ def shutdown @thread.join end end -=begin - def enumerate(eventList = nil) - currentTime = Time.now - emitTime = currentTime.to_f - batchTime = currentTime.utc.iso8601 - if eventList.nil? - $log.info("in_kube_events::enumerate : Getting events from Kube API @ #{Time.now.utc.iso8601}") - events = JSON.parse(KubernetesApiClient.getKubeResourceInfo('events').body) - $log.info("in_kube_events::enumerate : Done getting events from Kube API @ #{Time.now.utc.iso8601}") - else - events = eventList - end - eventQueryState = getEventQueryState - newEventQueryState = [] - begin - if(!events.empty?) - eventStream = MultiEventStream.new - events['items'].each do |items| - record = {} - record['CollectionTime'] = batchTime #This is the time that is mapped to become TimeGenerated - eventId = items['metadata']['uid'] + "/" + items['count'].to_s - newEventQueryState.push(eventId) - if !eventQueryState.empty? && eventQueryState.include?(eventId) - next - end - record['ObjectKind']= items['involvedObject']['kind'] - record['Namespace'] = items['involvedObject']['namespace'] - record['Name'] = items['involvedObject']['name'] - record['Reason'] = items['reason'] - record['Message'] = items['message'] - record['Type'] = items['type'] - record['TimeGenerated'] = items['metadata']['creationTimestamp'] - record['SourceComponent'] = items['source']['component'] - record['FirstSeen'] = items['firstTimestamp'] - record['LastSeen'] = items['lastTimestamp'] - record['Count'] = items['count'] - if items['source'].key?('host') - record['Computer'] = items['source']['host'] - else - record['Computer'] = (OMS::Common.get_hostname) - end - record['ClusterName'] = KubernetesApiClient.getClusterName - record['ClusterId'] = KubernetesApiClient.getClusterId - eventStream.add(emitTime, record) if record - end - router.emit_stream(@tag, eventStream) if eventStream - end - writeEventQueryState(newEventQueryState) - rescue => errorStr - $log.warn line.dump, error: errorStr.to_s - $log.debug_backtrace(errorStr.backtrace) - end + + def enumerate + currentTime = Time.now + emitTime = currentTime.to_f + batchTime = currentTime.utc.iso8601 + myhost = DockerApiClient.getDockerHostName + begin + eventStream = MultiEventStream.new + record = {} + record['myhost'] = myhost + eventStream.add(emitTime, record) if record + router.emit_stream(@tag, eventStream) if eventStream + end end -=end + def run_periodic @mutex.lock done = @finished @@ -104,11 +61,10 @@ def run_periodic @mutex.unlock if !done begin - $log.info("in_kube_events::run_periodic @ #{Time.now.utc.iso8601}") - myhost = DockerApiClient.getDockerHostName() - router.emit_stream(@tag, myhost) if myhost + $log.info("in_container_inventory::run_periodic @ #{Time.now.utc.iso8601}") + enumerate rescue => errorStr - $log.warn "in_kube_events::run_periodic: enumerate Failed to retrieve kube events: #{errorStr}" + $log.warn "in_container_inventory::run_periodic: enumerate Failed to retrieve docker container inventory: #{errorStr}" end end @mutex.lock @@ -116,7 +72,6 @@ def run_periodic @mutex.unlock end - end # Kube_Event_Input - -end # module + end # Container_Inventory_Input +end # module \ No newline at end of file From 34294c17f335a1a15022b553de2a935ffa3fa6e6 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 23 Oct 2018 15:02:44 -0700 Subject: [PATCH 07/45] changes --- source/code/plugin/DockerApiClient.rb | 51 ++++++++++++++++++-- source/code/plugin/in_container_inventory.rb | 7 ++- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index ba191b98d..c16689c33 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -54,7 +54,7 @@ def parseResponse(dockerResponse, isMultiJson) def getDockerHostName() dockerHostName = "" - request = DockerApiRestHelper.restDockerInfo() + request = DockerApiRestHelper.restDockerInfo response = getResponse(request, false) if (response != nil) dockerHostName = response['Name'] @@ -64,13 +64,56 @@ def getDockerHostName() def listContainers() ids = [] - request = DockerApiRestHelper.restDockerPs() + request = DockerApiRestHelper.restDockerPs containers = getResponse(request, true) - containers.each do |container| - ids.push container['Id'] + if !containers.nil? && !containers.empty? + containers.each do |container| + ids.push(container['Id']) + end end return ids end + + def getImageRepositoryImageTag(tagValue) + result = ["" "", ""] + if !tagValue.empty? + # Find delimiters in the string of format repository/image:imagetag + slashLocation = tagValue.index('/') + colonLocation = tagValue.index(':') + if !colonLocation.nil? + if slashLocation.nil? + # image:imagetag + result[1] = tagValue[0..(colonLocation-1)] + else + # repository/image:imagetag + result[0] = tagValue[0..(slashLocation-1)] + result[1] = tagValue[(slashLocation + 1)..(colonLocation - 1)] + end + result[2] = tagValue[(colonLocation + 1)..-1] + end + end + return result + end + + def generateImageNameMap() + result = nil + request = DockerApiRestHelper.restDockerImages + images = getResponse(request, true) + if !images.nil? && !images.empty? + images.each do |image| + tagValue = "" + tags = image['RepoTags'] + if !tags.nil? && tags.kind_of?(Array) && tags.length > 0 + tagValue = tags[0] + end + idValue = image['Id'] + if !idValue.nil? + result[idValue] = getImageRepositoryImageTag(tagValue) + end + end + end + return result + end end end diff --git a/source/code/plugin/in_container_inventory.rb b/source/code/plugin/in_container_inventory.rb index fe8f9be6d..56300131d 100644 --- a/source/code/plugin/in_container_inventory.rb +++ b/source/code/plugin/in_container_inventory.rb @@ -10,6 +10,7 @@ def initialize super require 'json' require_relative 'DockerApiClient' + require_relative 'omslog' end config_param :run_interval, :time, :default => '1m' @@ -42,13 +43,17 @@ def enumerate currentTime = Time.now emitTime = currentTime.to_f batchTime = currentTime.utc.iso8601 - myhost = DockerApiClient.getDockerHostName + $log.info("in_container_inventory::enumerate : Begin processing @ #{Time.now.utc.iso8601}") + hostname = DockerApiClient.getDockerHostName begin + containerIds = DockerApiClient.listContainers eventStream = MultiEventStream.new record = {} record['myhost'] = myhost eventStream.add(emitTime, record) if record router.emit_stream(@tag, eventStream) if eventStream + rescue => errorStr + end end From 7ce3d08111fa34607d7a9a443cd1b1b86dd816b0 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 23 Oct 2018 15:52:23 -0700 Subject: [PATCH 08/45] changes --- source/code/plugin/DockerApiClient.rb | 2 +- source/code/plugin/in_container_inventory.rb | 43 ++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index c16689c33..e63ccb65f 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -95,7 +95,7 @@ def getImageRepositoryImageTag(tagValue) return result end - def generateImageNameMap() + def getImageIdMap() result = nil request = DockerApiRestHelper.restDockerImages images = getResponse(request, true) diff --git a/source/code/plugin/in_container_inventory.rb b/source/code/plugin/in_container_inventory.rb index 56300131d..6a68402c7 100644 --- a/source/code/plugin/in_container_inventory.rb +++ b/source/code/plugin/in_container_inventory.rb @@ -39,6 +39,39 @@ def shutdown end end + def inspectContainer(id, nameMap) + containerInstance = {} + request = DockerApiRestHelper.restDockerInspect(id) + container = getResponse(request, false) + if !container.nil? && !container.empty? + containerInstance['InstanceID'] = container['Id'] + containerInstance['CreatedTime'] = container['Created'] + containerName = container['Name'] + if !containerName.nil? && !containerName.empty? + # Remove the leading / from the name if it exists (this is an API issue) + containerInstance['ElementName'] = (containerName[0] == '/') ? containerName[1..-1] : containerName + end + imageValue = container['Image'] + if !imageValue.nil? && !imageValue.empty? + containerInstance['ImageId'] = imageValue + repoImageTagArray = nameMap['imageValue'] + if nameMap.has_key? imageValue + containerInstance['Repository'] = repoImageTagArray[0] + containerInstance['Image'] = repoImageTagArray[1] + containerInstance['ImageTag'] = repoImageTagArray[2] + end + end + + + + + + end + + + end + + def enumerate currentTime = Time.now emitTime = currentTime.to_f @@ -47,6 +80,16 @@ def enumerate hostname = DockerApiClient.getDockerHostName begin containerIds = DockerApiClient.listContainers + nameMap = DockerApiClient.getImageIdMap + containerIds.each do |containerId| + inspectedContainer = {} + inspectedContainer = inspectContainer(containerId, nameMap) + inspectedContainer['Computer'] = hostname + end + + + + eventStream = MultiEventStream.new record = {} record['myhost'] = myhost From 24d912c37448d677e08f17d951540b4b7640a76a Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 23 Oct 2018 18:40:10 -0700 Subject: [PATCH 09/45] changes --- source/code/plugin/in_container_inventory.rb | 73 ++++++++++++++++++-- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/source/code/plugin/in_container_inventory.rb b/source/code/plugin/in_container_inventory.rb index 6a68402c7..d2db02964 100644 --- a/source/code/plugin/in_container_inventory.rb +++ b/source/code/plugin/in_container_inventory.rb @@ -39,6 +39,71 @@ def shutdown end end + def obtainContainerConfig(instance, container) + configValue = container['Config'] + if !configValue.nil? + instance['ContainerHostname'] = configValue['Hostname'] +=begin envValue = "" + envItem = configValue['Env'] + if !envItem.nil? && !envItem.empty? + envStringLength = envItem.length + if envStringLength > 200000 + quotestring = "\""; + quoteandbracestring = "\"]"; + quoteandcommastring = "\","; + stringToTruncate = envItem[0..200000]; + end + end +=end + instance['EnvironmentVar'] = configValue['Env'] + instance['Command'] = configValue['Cmd'] + labelsValue = configValue['Labels'] + if !labelsValue.nil? && !labelsValue.empty? + instance['ComposeGroup'] = labelsValue['com.docker.compose.project'] + end + end + end + + def obtainContainerState(instance, container) + stateValue = container['State'] + if !stateValue.nil? + exitCode = stateValue['ExitCode'] + if exitCode .is_a? Numeric + if exitCode < 0 + exitCode = 128 + $log.info("obtainContainerState::Container #{container['Id']} returned negative exit code") + end + instance['ExitCode'] = exitCode + if exitCode > 0 + instance['State'] = "Failed" + else + + end + + + end + end + + end + + def obtainContainerHostConfig(instance, container) + hostConfig = container['HostConfig'] + if !hostConfig.nil? + links = hostConfig['Links'] + if !links.nil? + links.to_s + instance['Links'] = (links == "null")? "" : links + end + portBindings = hostConfig['PortBindings'] + if !portBindings.nil? + portBindings.to_s + instance['Ports'] = (portBindings == "null")? "" : portBindings + end + end + end + + + def inspectContainer(id, nameMap) containerInstance = {} request = DockerApiRestHelper.restDockerInspect(id) @@ -61,11 +126,9 @@ def inspectContainer(id, nameMap) containerInstance['ImageTag'] = repoImageTagArray[2] end end - - - - - + obtainContainerConfig(containerInstance, container); + obtainContainerState(containerInstance, container); + obtainContainerHostConfig(containerInstance, container); end From 07de7e5fb6ddde0653cdcf39077f859b460ae7fc Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 24 Oct 2018 12:01:44 -0700 Subject: [PATCH 10/45] changes --- source/code/plugin/in_container_inventory.rb | 76 ++++++++++++-------- 1 file changed, 45 insertions(+), 31 deletions(-) diff --git a/source/code/plugin/in_container_inventory.rb b/source/code/plugin/in_container_inventory.rb index d2db02964..4e91bfe85 100644 --- a/source/code/plugin/in_container_inventory.rb +++ b/source/code/plugin/in_container_inventory.rb @@ -40,50 +40,66 @@ def shutdown end def obtainContainerConfig(instance, container) + begin configValue = container['Config'] if !configValue.nil? instance['ContainerHostname'] = configValue['Hostname'] -=begin envValue = "" - envItem = configValue['Env'] - if !envItem.nil? && !envItem.empty? - envStringLength = envItem.length - if envStringLength > 200000 - quotestring = "\""; - quoteandbracestring = "\"]"; - quoteandcommastring = "\","; - stringToTruncate = envItem[0..200000]; - end - end -=end - instance['EnvironmentVar'] = configValue['Env'] - instance['Command'] = configValue['Cmd'] + + envValue = configValue['Env'] + envValueString = (envValue.nil?) ? "" : envValue.to_s + instance['EnvironmentVar'] = envValueString + + cmdValue = configValue['Cmd'] + cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s + instance['Command'] = cmdValueString + + instance['ComposeGroup'] = "" labelsValue = configValue['Labels'] if !labelsValue.nil? && !labelsValue.empty? instance['ComposeGroup'] = labelsValue['com.docker.compose.project'] end + else + $log.warn("Attempt in ObtainContainerConfig to get container #{container['Id']} config information returned null") + end + rescue => errorStr + $log.warn("Exception in obtainContainerConfig: #{errorStr}") end end def obtainContainerState(instance, container) + begin stateValue = container['State'] if !stateValue.nil? - exitCode = stateValue['ExitCode'] - if exitCode .is_a? Numeric - if exitCode < 0 - exitCode = 128 - $log.info("obtainContainerState::Container #{container['Id']} returned negative exit code") - end - instance['ExitCode'] = exitCode - if exitCode > 0 - instance['State'] = "Failed" + exitCodeValue = stateValue['ExitCode'] + # Exit codes less than 0 are not supported by the engine + if exitCodeValue < 0 + exitCodeValue = 128 + $log.info("obtainContainerState::Container #{container['Id']} returned negative exit code") + end + instance['ExitCode'] = exitCodeValue + if exitCodeValue > 0 + instance['State'] = "Failed" + else + # Set the Container status : Running/Paused/Stopped + runningValue = stateValue['Running'] + if runningValue + pausedValue = stateValue['Paused'] + # Checking for paused within running is true state because docker returns true for both Running and Paused fields when the container is paused + if pausedValue + instance['State'] = "Paused" + else + instance['State'] = "Running" + end else - + instance['State'] = "Stopped" end - - end + else + $log.info("Attempt in ObtainContainerState to get container: #{container['Id']} state information returned null") + end + rescue => errorStr + $log.warn("Exception in obtainContainerState: #{errorStr}") end - end def obtainContainerHostConfig(instance, container) @@ -91,8 +107,8 @@ def obtainContainerHostConfig(instance, container) if !hostConfig.nil? links = hostConfig['Links'] if !links.nil? - links.to_s - instance['Links'] = (links == "null")? "" : links + linksString = links.to_s + instance['Links'] = (linksString == "null")? "" : links end portBindings = hostConfig['PortBindings'] if !portBindings.nil? @@ -102,8 +118,6 @@ def obtainContainerHostConfig(instance, container) end end - - def inspectContainer(id, nameMap) containerInstance = {} request = DockerApiRestHelper.restDockerInspect(id) From 73877ff350a26f8dee1e0124a05005e339d504cc Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 24 Oct 2018 12:06:47 -0700 Subject: [PATCH 11/45] changes --- source/code/plugin/in_container_inventory.rb | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/source/code/plugin/in_container_inventory.rb b/source/code/plugin/in_container_inventory.rb index 4e91bfe85..f16f2accc 100644 --- a/source/code/plugin/in_container_inventory.rb +++ b/source/code/plugin/in_container_inventory.rb @@ -103,18 +103,24 @@ def obtainContainerState(instance, container) end def obtainContainerHostConfig(instance, container) + begin hostConfig = container['HostConfig'] if !hostConfig.nil? links = hostConfig['Links'] if !links.nil? linksString = links.to_s - instance['Links'] = (linksString == "null")? "" : links + instance['Links'] = (linksString == "null")? "" : linksString end portBindings = hostConfig['PortBindings'] if !portBindings.nil? - portBindings.to_s - instance['Ports'] = (portBindings == "null")? "" : portBindings + portBindingsString = portBindings.to_s + instance['Ports'] = (portBindings == "null")? "" : portBindingsString end + else + $log.info("Attempt in ObtainContainerHostConfig to get container: #{container['Id']} host config information returned null") + end + rescue => errorStr + $log.warn("Exception in obtainContainerHostConfig: #{errorStr}") end end From ca3ee04d3dd0afedba9b04496196342e9a5ff0fc Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 24 Oct 2018 17:39:01 -0700 Subject: [PATCH 12/45] working containerinventory --- installer/conf/container.conf | 51 +---- source/code/plugin/DockerApiClient.rb | 46 ++-- source/code/plugin/DockerApiRestHelper.rb | 9 - source/code/plugin/in_container_inventory.rb | 208 ----------------- source/code/plugin/in_containerinventory.rb | 224 +++++++++++++++++++ 5 files changed, 257 insertions(+), 281 deletions(-) delete mode 100644 source/code/plugin/in_container_inventory.rb create mode 100644 source/code/plugin/in_containerinventory.rb diff --git a/installer/conf/container.conf b/installer/conf/container.conf index bb341e33b..dcecc7477 100755 --- a/installer/conf/container.conf +++ b/installer/conf/container.conf @@ -8,15 +8,6 @@ # Container inventory - - type omi - run_interval 60s - tag oms.container.containerinventory - items [ - ["root/cimv2","Container_ContainerInventory"] - ] - - type containerinventory tag oms.containerinsights.ContainerInventory @@ -24,16 +15,6 @@ log_level debug -# Image inventory - - type omi - run_interval 60s - tag oms.container.imageinventory - items [ - ["root/cimv2","Container_ImageInventory"] - ] - - # Container host inventory type omi @@ -52,15 +33,6 @@ log_level debug -# Filter for correct format to endpoint - - type filter_container - - - - @type stdout - - type out_oms_api log_level debug @@ -74,33 +46,22 @@ max_retry_wait 9m - - type out_oms - log_level debug - buffer_chunk_limit 20m - buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_containerinventory*.buffer - buffer_queue_limit 20 - flush_interval 20s - retry_limit 10 - retry_wait 15s - max_retry_wait 9m - - - + type out_oms log_level debug + num_threads 5 buffer_chunk_limit 20m buffer_type file - buffer_path %STATE_DIR_WS%/out_oms_imageinventory*.buffer + buffer_path /var/opt/microsoft/omsagent/62a84175-b18d-48ff-8b42-0694dda7b8bf/state/state/out_oms_containerinventory*.buffer buffer_queue_limit 20 + buffer_queue_full_action drop_oldest_chunk flush_interval 20s retry_limit 10 - retry_wait 15s + retry_wait 30s max_retry_wait 9m - + type out_oms log_level debug num_threads 5 diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index e63ccb65f..a6552ed5f 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -18,18 +18,22 @@ def initialize class << self # Make docker socket call to def getResponse(request, isMultiJson) - #TODO: Add error handling and retries - socket = UNIXSocket.new(@@SocketPath) - dockerResponse = "" - socket.write(request) - # iterate through the response until the last chunk is less than the chunk size so that we can read all data in socket. - loop do - responseChunk = socket.recv(@@ChunkSize) - dockerResponse += responseChunk - break if responseChunk.length < @@ChunkSize + #TODO: Add retries + begin + socket = UNIXSocket.new(@@SocketPath) + dockerResponse = "" + socket.write(request) + # iterate through the response until the last chunk is less than the chunk size so that we can read all data in socket. + loop do + responseChunk = socket.recv(@@ChunkSize) + dockerResponse += responseChunk + break if responseChunk.length < @@ChunkSize + end + socket.close + return parseResponse(dockerResponse, isMultiJson) + rescue => errorStr + $log.warn("Socket call failed for request: #{request} error: #{errorStr} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") end - socket.close - return parseResponse(dockerResponse, isMultiJson) end def parseResponse(dockerResponse, isMultiJson) @@ -38,15 +42,15 @@ def parseResponse(dockerResponse, isMultiJson) parsedJsonResponse = nil begin jsonResponse = isMultiJson ? dockerResponse[/\[{.+}\]/] : dockerResponse[/{.+}/] - rescue => exception - @Log.warn("Regex match for docker response failed: #{exception} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") + rescue => errorStr + $log.warn("Regex match for docker response failed: #{errorStr} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") end begin if jsonResponse != nil parsedJsonResponse = JSON.parse(jsonResponse) end - rescue => exception - @Log.warn("Json parsing for docker response failed: #{exception} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") + rescue => errorStr + $log.warn("Json parsing for docker response failed: #{errorStr} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") end return parsedJsonResponse end @@ -75,7 +79,7 @@ def listContainers() end def getImageRepositoryImageTag(tagValue) - result = ["" "", ""] + result = ["", "", ""] if !tagValue.empty? # Find delimiters in the string of format repository/image:imagetag slashLocation = tagValue.index('/') @@ -100,6 +104,7 @@ def getImageIdMap() request = DockerApiRestHelper.restDockerImages images = getResponse(request, true) if !images.nil? && !images.empty? + result = {} images.each do |image| tagValue = "" tags = image['RepoTags'] @@ -114,7 +119,10 @@ def getImageIdMap() end return result end - end -end - \ No newline at end of file + def dockerInspectContainer(id) + request = DockerApiRestHelper.restDockerInspect(id) + return getResponse(request, false) + end + end +end \ No newline at end of file diff --git a/source/code/plugin/DockerApiRestHelper.rb b/source/code/plugin/DockerApiRestHelper.rb index 8a1e1585e..b9c9304eb 100644 --- a/source/code/plugin/DockerApiRestHelper.rb +++ b/source/code/plugin/DockerApiRestHelper.rb @@ -24,15 +24,6 @@ def restDockerPs() end end - # Create the REST request to list running containers - # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.21/#list-containers - # returns Request in string format - def restDockerPsRunning() - begin - return "GET /containers/json HTTP/1.1\r\nHost: localhost\r\n\r\n"; - end - end - # Create the REST request to inspect a container # https://docs.docker.com/engine/reference/api/docker_remote_api_v1.21/#inspect-a-container # parameter - ID of the container to be inspected diff --git a/source/code/plugin/in_container_inventory.rb b/source/code/plugin/in_container_inventory.rb deleted file mode 100644 index f16f2accc..000000000 --- a/source/code/plugin/in_container_inventory.rb +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/local/bin/ruby -# frozen_string_literal: true - -module Fluent - - class Container_Inventory_Input < Input - Plugin.register_input('containerinventory', self) - - def initialize - super - require 'json' - require_relative 'DockerApiClient' - require_relative 'omslog' - end - - config_param :run_interval, :time, :default => '1m' - config_param :tag, :string, :default => "oms.containerinsights.ContainerInventory" - - def configure (conf) - super - end - - def start - if @run_interval - @finished = false - @condition = ConditionVariable.new - @mutex = Mutex.new - @thread = Thread.new(&method(:run_periodic)) - end - end - - def shutdown - if @run_interval - @mutex.synchronize { - @finished = true - @condition.signal - } - @thread.join - end - end - - def obtainContainerConfig(instance, container) - begin - configValue = container['Config'] - if !configValue.nil? - instance['ContainerHostname'] = configValue['Hostname'] - - envValue = configValue['Env'] - envValueString = (envValue.nil?) ? "" : envValue.to_s - instance['EnvironmentVar'] = envValueString - - cmdValue = configValue['Cmd'] - cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s - instance['Command'] = cmdValueString - - instance['ComposeGroup'] = "" - labelsValue = configValue['Labels'] - if !labelsValue.nil? && !labelsValue.empty? - instance['ComposeGroup'] = labelsValue['com.docker.compose.project'] - end - else - $log.warn("Attempt in ObtainContainerConfig to get container #{container['Id']} config information returned null") - end - rescue => errorStr - $log.warn("Exception in obtainContainerConfig: #{errorStr}") - end - end - - def obtainContainerState(instance, container) - begin - stateValue = container['State'] - if !stateValue.nil? - exitCodeValue = stateValue['ExitCode'] - # Exit codes less than 0 are not supported by the engine - if exitCodeValue < 0 - exitCodeValue = 128 - $log.info("obtainContainerState::Container #{container['Id']} returned negative exit code") - end - instance['ExitCode'] = exitCodeValue - if exitCodeValue > 0 - instance['State'] = "Failed" - else - # Set the Container status : Running/Paused/Stopped - runningValue = stateValue['Running'] - if runningValue - pausedValue = stateValue['Paused'] - # Checking for paused within running is true state because docker returns true for both Running and Paused fields when the container is paused - if pausedValue - instance['State'] = "Paused" - else - instance['State'] = "Running" - end - else - instance['State'] = "Stopped" - end - end - else - $log.info("Attempt in ObtainContainerState to get container: #{container['Id']} state information returned null") - end - rescue => errorStr - $log.warn("Exception in obtainContainerState: #{errorStr}") - end - end - - def obtainContainerHostConfig(instance, container) - begin - hostConfig = container['HostConfig'] - if !hostConfig.nil? - links = hostConfig['Links'] - if !links.nil? - linksString = links.to_s - instance['Links'] = (linksString == "null")? "" : linksString - end - portBindings = hostConfig['PortBindings'] - if !portBindings.nil? - portBindingsString = portBindings.to_s - instance['Ports'] = (portBindings == "null")? "" : portBindingsString - end - else - $log.info("Attempt in ObtainContainerHostConfig to get container: #{container['Id']} host config information returned null") - end - rescue => errorStr - $log.warn("Exception in obtainContainerHostConfig: #{errorStr}") - end - end - - def inspectContainer(id, nameMap) - containerInstance = {} - request = DockerApiRestHelper.restDockerInspect(id) - container = getResponse(request, false) - if !container.nil? && !container.empty? - containerInstance['InstanceID'] = container['Id'] - containerInstance['CreatedTime'] = container['Created'] - containerName = container['Name'] - if !containerName.nil? && !containerName.empty? - # Remove the leading / from the name if it exists (this is an API issue) - containerInstance['ElementName'] = (containerName[0] == '/') ? containerName[1..-1] : containerName - end - imageValue = container['Image'] - if !imageValue.nil? && !imageValue.empty? - containerInstance['ImageId'] = imageValue - repoImageTagArray = nameMap['imageValue'] - if nameMap.has_key? imageValue - containerInstance['Repository'] = repoImageTagArray[0] - containerInstance['Image'] = repoImageTagArray[1] - containerInstance['ImageTag'] = repoImageTagArray[2] - end - end - obtainContainerConfig(containerInstance, container); - obtainContainerState(containerInstance, container); - obtainContainerHostConfig(containerInstance, container); - end - - - end - - - def enumerate - currentTime = Time.now - emitTime = currentTime.to_f - batchTime = currentTime.utc.iso8601 - $log.info("in_container_inventory::enumerate : Begin processing @ #{Time.now.utc.iso8601}") - hostname = DockerApiClient.getDockerHostName - begin - containerIds = DockerApiClient.listContainers - nameMap = DockerApiClient.getImageIdMap - containerIds.each do |containerId| - inspectedContainer = {} - inspectedContainer = inspectContainer(containerId, nameMap) - inspectedContainer['Computer'] = hostname - end - - - - - eventStream = MultiEventStream.new - record = {} - record['myhost'] = myhost - eventStream.add(emitTime, record) if record - router.emit_stream(@tag, eventStream) if eventStream - rescue => errorStr - - end - end - - def run_periodic - @mutex.lock - done = @finished - until done - @condition.wait(@mutex, @run_interval) - done = @finished - @mutex.unlock - if !done - begin - $log.info("in_container_inventory::run_periodic @ #{Time.now.utc.iso8601}") - enumerate - rescue => errorStr - $log.warn "in_container_inventory::run_periodic: enumerate Failed to retrieve docker container inventory: #{errorStr}" - end - end - @mutex.lock - end - @mutex.unlock - end - - end # Container_Inventory_Input - -end # module \ No newline at end of file diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb new file mode 100644 index 000000000..490e92b65 --- /dev/null +++ b/source/code/plugin/in_containerinventory.rb @@ -0,0 +1,224 @@ +#!/usr/local/bin/ruby +# frozen_string_literal: true + +module Fluent + + class Container_Inventory_Input < Input + Plugin.register_input('containerinventory', self) + + def initialize + super + require 'json' + require_relative 'DockerApiClient' + require_relative 'omslog' + end + + config_param :run_interval, :time, :default => '1m' + config_param :tag, :string, :default => "oms.containerinsights.ContainerInventory" + + def configure (conf) + super + end + + def start + if @run_interval + @finished = false + @condition = ConditionVariable.new + @mutex = Mutex.new + @thread = Thread.new(&method(:run_periodic)) + end + end + + def shutdown + if @run_interval + @mutex.synchronize { + @finished = true + @condition.signal + } + @thread.join + end + end + + def obtainContainerConfig(instance, container) + begin + configValue = container['Config'] + if !configValue.nil? + instance['ContainerHostname'] = configValue['Hostname'] + + envValue = configValue['Env'] + envValueString = (envValue.nil?) ? "" : envValue.to_s + instance['EnvironmentVar'] = envValueString + + cmdValue = configValue['Cmd'] + cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s + instance['Command'] = cmdValueString + + instance['ComposeGroup'] = "" + labelsValue = configValue['Labels'] + if !labelsValue.nil? && !labelsValue.empty? + instance['ComposeGroup'] = labelsValue['com.docker.compose.project'] + end + else + $log.warn("Attempt in ObtainContainerConfig to get container #{container['Id']} config information returned null") + end + rescue => errorStr + $log.warn("Exception in obtainContainerConfig: #{errorStr}") + end + end + + def obtainContainerState(instance, container) + begin + stateValue = container['State'] + if !stateValue.nil? + exitCodeValue = stateValue['ExitCode'] + # Exit codes less than 0 are not supported by the engine + if exitCodeValue < 0 + exitCodeValue = 128 + $log.info("obtainContainerState::Container #{container['Id']} returned negative exit code") + end + instance['ExitCode'] = exitCodeValue + if exitCodeValue > 0 + instance['State'] = "Failed" + else + # Set the Container status : Running/Paused/Stopped + runningValue = stateValue['Running'] + if runningValue + pausedValue = stateValue['Paused'] + # Checking for paused within running is true state because docker returns true for both Running and Paused fields when the container is paused + if pausedValue + instance['State'] = "Paused" + else + instance['State'] = "Running" + end + else + instance['State'] = "Stopped" + end + end + instance['StartedTime'] = stateValue['StartedAt'] + instance['FinishedTime'] = stateValue['FinishedAt'] + else + $log.info("Attempt in ObtainContainerState to get container: #{container['Id']} state information returned null") + end + rescue => errorStr + $log.warn("Exception in obtainContainerState: #{errorStr}") + end + end + + def obtainContainerHostConfig(instance, container) + begin + hostConfig = container['HostConfig'] + if !hostConfig.nil? + links = hostConfig['Links'] + instance['Links'] = "" + if !links.nil? + linksString = links.to_s + instance['Links'] = (linksString == "null")? "" : linksString + end + portBindings = hostConfig['PortBindings'] + instance['Ports'] = "" + if !portBindings.nil? + portBindingsString = portBindings.to_s + instance['Ports'] = (portBindingsString == "null")? "" : portBindingsString + end + else + $log.info("Attempt in ObtainContainerHostConfig to get container: #{container['Id']} host config information returned null") + end + rescue => errorStr + $log.warn("Exception in obtainContainerHostConfig: #{errorStr}") + end + end + + def inspectContainer(id, nameMap) + containerInstance = {} + begin + container = DockerApiClient.dockerInspectContainer(id) + if !container.nil? && !container.empty? + containerInstance['InstanceID'] = "rashmi" + container['Id'] + containerInstance['CreatedTime'] = container['Created'] + containerName = container['Name'] + if !containerName.nil? && !containerName.empty? + # Remove the leading / from the name if it exists (this is an API issue) + containerInstance['ElementName'] = (containerName[0] == '/') ? containerName[1..-1] : containerName + end + imageValue = container['Image'] + if !imageValue.nil? && !imageValue.empty? + containerInstance['ImageId'] = imageValue + repoImageTagArray = nameMap[imageValue] + if nameMap.has_key? imageValue + containerInstance['Repository'] = repoImageTagArray[0] + containerInstance['Image'] = repoImageTagArray[1] + containerInstance['ImageTag'] = repoImageTagArray[2] + end + end + obtainContainerConfig(containerInstance, container); + obtainContainerState(containerInstance, container); + obtainContainerHostConfig(containerInstance, container); + end + rescue => errorStr + $log.warn("Exception in inspectContainer: #{errorStr} for container: #{id}") + end + return containerInstance + end + + def enumerate + currentTime = Time.now + emitTime = currentTime.to_f + batchTime = currentTime.utc.iso8601 + containerInventory = Array.new + $log.info("in_container_inventory::enumerate : Begin processing @ #{Time.now.utc.iso8601}") + hostname = DockerApiClient.getDockerHostName + $log.info("Done getting host name @ #{Time.now.utc.iso8601}") + begin + containerIds = DockerApiClient.listContainers + $log.info("Done getting containers @ #{Time.now.utc.iso8601}") + if !containerIds.empty? + $log.info("in container ids not empty @ #{Time.now.utc.iso8601}") + eventStream = MultiEventStream.new + $log.info("Done creating eventstream @ #{Time.now.utc.iso8601}") + nameMap = DockerApiClient.getImageIdMap + $log.info("Done getting namemap @ #{Time.now.utc.iso8601}") + containerIds.each do |containerId| + inspectedContainer = {} + inspectedContainer = inspectContainer(containerId, nameMap) + inspectedContainer['Computer'] = hostname + containerInventory.push inspectedContainer + end + #TODO: Get deleted container state and update it + containerInventory.each do |record| + wrapper = { + "DataType"=>"CONTAINER_INVENTORY_BLOB", + "IPName"=>"ContainerInsights", + "DataItems"=>[record.each{|k,v| record[k]=v}] + } + eventStream.add(emitTime, wrapper) if wrapper + end + router.emit_stream(@tag, eventStream) if eventStream + end + rescue => errorStr + $log.warn("Exception in enumerate container inventory: #{errorStr}") + end + end + + def run_periodic + @mutex.lock + done = @finished + until done + @condition.wait(@mutex, @run_interval) + done = @finished + @mutex.unlock + if !done + begin + $log.info("in_container_inventory::run_periodic @ #{Time.now.utc.iso8601}") + enumerate + rescue => errorStr + $log.warn "in_container_inventory::run_periodic: Failed in enumerate container inventory: #{errorStr}" + end + end + @mutex.lock + end + @mutex.unlock + end + + end # Container_Inventory_Input + +end # module \ No newline at end of file From 1492dabe9d716af7dd04441e20572a03c4817b69 Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 24 Oct 2018 17:53:03 -0700 Subject: [PATCH 13/45] fixing omi removal from container.conf --- installer/conf/container.conf | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/installer/conf/container.conf b/installer/conf/container.conf index dcecc7477..cc01244c9 100755 --- a/installer/conf/container.conf +++ b/installer/conf/container.conf @@ -7,6 +7,12 @@ bind 127.0.0.1 + + type oms_omi + object_name "Container" + interval 30s + + # Container inventory type containerinventory From 8d4ca54ea6d71aa4ac19a0b44d066579f0e70447 Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 24 Oct 2018 17:56:59 -0700 Subject: [PATCH 14/45] removing comments --- source/code/plugin/in_containerinventory.rb | 5 ----- 1 file changed, 5 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 490e92b65..dfa230e33 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -167,16 +167,11 @@ def enumerate containerInventory = Array.new $log.info("in_container_inventory::enumerate : Begin processing @ #{Time.now.utc.iso8601}") hostname = DockerApiClient.getDockerHostName - $log.info("Done getting host name @ #{Time.now.utc.iso8601}") begin containerIds = DockerApiClient.listContainers - $log.info("Done getting containers @ #{Time.now.utc.iso8601}") if !containerIds.empty? - $log.info("in container ids not empty @ #{Time.now.utc.iso8601}") eventStream = MultiEventStream.new - $log.info("Done creating eventstream @ #{Time.now.utc.iso8601}") nameMap = DockerApiClient.getImageIdMap - $log.info("Done getting namemap @ #{Time.now.utc.iso8601}") containerIds.each do |containerId| inspectedContainer = {} inspectedContainer = inspectContainer(containerId, nameMap) From a8e3498734069b3718d01bddc758b55062faa873 Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 25 Oct 2018 12:55:39 -0700 Subject: [PATCH 15/45] file write and read --- source/code/plugin/ContainerInventoryState.rb | 36 +++++++++++++++++++ source/code/plugin/in_containerinventory.rb | 2 ++ 2 files changed, 38 insertions(+) create mode 100644 source/code/plugin/ContainerInventoryState.rb diff --git a/source/code/plugin/ContainerInventoryState.rb b/source/code/plugin/ContainerInventoryState.rb new file mode 100644 index 000000000..4343dc29a --- /dev/null +++ b/source/code/plugin/ContainerInventoryState.rb @@ -0,0 +1,36 @@ +#!/usr/local/bin/ruby +# frozen_string_literal: true + +class ContainerInventoryState + require_relative 'oms_common' + require_relative 'omslog' + @@InventoryDirectory = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory/" + + def initialize + end + + class << self + def WriteContainerState(container) + containerId = container['InstanceID'] + if !containerId.nil? && !containerId.empty? + begin + file = File.open(@@InventoryDirectory + containerId, "w") + if !file.nil? + file.write(container.to_json) + file.close + else + $log.warn("Exception top open file with id: #{containerId}") + end + rescue => errorStr + $log.warn("Exception in WriteContainerState: #{errorStr}") + end + end + end + + def ReadContainerState(containerId) + + end + end +end + + \ No newline at end of file diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index dfa230e33..549888cd9 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -10,6 +10,7 @@ def initialize super require 'json' require_relative 'DockerApiClient' + require_relative 'ContainerInventoryState' require_relative 'omslog' end @@ -177,6 +178,7 @@ def enumerate inspectedContainer = inspectContainer(containerId, nameMap) inspectedContainer['Computer'] = hostname containerInventory.push inspectedContainer + ContainerInventoryState.WriteContainerState(inspectedContainer) end #TODO: Get deleted container state and update it containerInventory.each do |record| From d0fbccbed94acda082ec7db3282186893314c246 Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 25 Oct 2018 15:25:49 -0700 Subject: [PATCH 16/45] deleted containers working --- source/code/plugin/ContainerInventoryState.rb | 35 +++++++++++++++---- source/code/plugin/in_containerinventory.rb | 18 ++++++++-- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/source/code/plugin/ContainerInventoryState.rb b/source/code/plugin/ContainerInventoryState.rb index 4343dc29a..50094c6b2 100644 --- a/source/code/plugin/ContainerInventoryState.rb +++ b/source/code/plugin/ContainerInventoryState.rb @@ -2,6 +2,7 @@ # frozen_string_literal: true class ContainerInventoryState + require 'json' require_relative 'oms_common' require_relative 'omslog' @@InventoryDirectory = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory/" @@ -10,7 +11,7 @@ def initialize end class << self - def WriteContainerState(container) + def writeContainerState(container) containerId = container['InstanceID'] if !containerId.nil? && !containerId.empty? begin @@ -19,7 +20,7 @@ def WriteContainerState(container) file.write(container.to_json) file.close else - $log.warn("Exception top open file with id: #{containerId}") + $log.warn("Exception while opening file with id: #{containerId}") end rescue => errorStr $log.warn("Exception in WriteContainerState: #{errorStr}") @@ -27,10 +28,32 @@ def WriteContainerState(container) end end - def ReadContainerState(containerId) + def readContainerState(containerId) + begin + containerObject = nil + file = File.open(@@InventoryDirectory + containerId, "r") + if !file.nil? + fileContents = file.read + containerObject = JSON.parse(fileContents) + file.close + else + $log.warn("Exception while opening file with id: #{containerId}") + end + rescue => errorStr + $log.warn("Exception in ReadContainerState: #{errorStr}") + end + return containerObject + end + def getDeletedContainers(containerIds) + deletedContainers = nil + begin + previousContainerList = Dir.entries(@@InventoryDirectory) - [".", ".."] + deletedContainers = previousContainerList - containerIds + rescue => errorStr + $log.warn("Exception in getDeletedContainers: #{errorStr}") + end + return deletedContainers end end -end - - \ No newline at end of file +end \ No newline at end of file diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 549888cd9..9d9c92def 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -134,7 +134,7 @@ def inspectContainer(id, nameMap) begin container = DockerApiClient.dockerInspectContainer(id) if !container.nil? && !container.empty? - containerInstance['InstanceID'] = "rashmi" + container['Id'] + containerInstance['InstanceID'] = container['Id'] containerInstance['CreatedTime'] = container['Created'] containerName = container['Name'] if !containerName.nil? && !containerName.empty? @@ -178,9 +178,21 @@ def enumerate inspectedContainer = inspectContainer(containerId, nameMap) inspectedContainer['Computer'] = hostname containerInventory.push inspectedContainer - ContainerInventoryState.WriteContainerState(inspectedContainer) + ContainerInventoryState.writeContainerState(inspectedContainer) end - #TODO: Get deleted container state and update it + # Update the state for deleted containers + deletedContainers = ContainerInventoryState.getDeletedContainers(containerIds) + if !deletedContainers.nil? && !deletedContainers.empty? + deletedContainers.each do |deletedContainer| + container = ContainerInventoryState.readContainerState(deletedContainer) + if !container.nil? + container.each{|k,v| container[k]=v} + container['State'] = "Deleted" + containerInventory.push container + end + end + end + containerInventory.each do |record| wrapper = { "DataType"=>"CONTAINER_INVENTORY_BLOB", From 4efc64806a598da465a4fabc53a00305394b3960 Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 25 Oct 2018 17:40:17 -0700 Subject: [PATCH 17/45] changes --- installer/datafiles/base_container.data | 3 +-- source/code/plugin/ContainerInventoryState.rb | 11 +++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/installer/datafiles/base_container.data b/installer/datafiles/base_container.data index 85a128b2a..610e642f7 100644 --- a/installer/datafiles/base_container.data +++ b/installer/datafiles/base_container.data @@ -74,8 +74,7 @@ MAINTAINER: 'Microsoft Corporation' /var/opt/microsoft; 755; root; root; sysdir /var/opt/microsoft/docker-cimprov; 755; root; root /var/opt/microsoft/docker-cimprov/state; 755; root; root -/var/opt/microsoft/docker-cimprov/state/ContainerInventory; 755; root; root -/var/opt/microsoft/docker-cimprov/state/ImageInventory; 755; root; root +/var/opt/microsoft/docker-cimprov/state/ContainerInventory; 755; omsagent; omsagent /var/opt/microsoft/docker-cimprov/log; 755; root; root /opt/td-agent-bit; 755; root; root;sysdir diff --git a/source/code/plugin/ContainerInventoryState.rb b/source/code/plugin/ContainerInventoryState.rb index 50094c6b2..a42c8e7b9 100644 --- a/source/code/plugin/ContainerInventoryState.rb +++ b/source/code/plugin/ContainerInventoryState.rb @@ -23,7 +23,7 @@ def writeContainerState(container) $log.warn("Exception while opening file with id: #{containerId}") end rescue => errorStr - $log.warn("Exception in WriteContainerState: #{errorStr}") + $log.warn("Exception in writeContainerState: #{errorStr}") end end end @@ -31,16 +31,19 @@ def writeContainerState(container) def readContainerState(containerId) begin containerObject = nil - file = File.open(@@InventoryDirectory + containerId, "r") + filepath = @@InventoryDirectory + containerId + file = File.open(filepath, "r") if !file.nil? fileContents = file.read containerObject = JSON.parse(fileContents) file.close + # Delete the file since the state is update to deleted + File.delete(filepath) if File.exist?(filepath) else - $log.warn("Exception while opening file with id: #{containerId}") + $log.warn("Open file for container with id returned nil: #{containerId}") end rescue => errorStr - $log.warn("Exception in ReadContainerState: #{errorStr}") + $log.warn("Exception in readContainerState: #{errorStr}") end return containerObject end From 44b40708353f5c1c160cfd7ec4c013858b06f6ef Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 25 Oct 2018 18:01:19 -0700 Subject: [PATCH 18/45] changes --- source/code/plugin/ContainerInventoryState.rb | 1 - source/code/plugin/DockerApiClient.rb | 61 +++++++++++-------- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/source/code/plugin/ContainerInventoryState.rb b/source/code/plugin/ContainerInventoryState.rb index a42c8e7b9..8eaf47e1d 100644 --- a/source/code/plugin/ContainerInventoryState.rb +++ b/source/code/plugin/ContainerInventoryState.rb @@ -3,7 +3,6 @@ class ContainerInventoryState require 'json' - require_relative 'oms_common' require_relative 'omslog' @@InventoryDirectory = "/var/opt/microsoft/docker-cimprov/state/ContainerInventory/" diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index a6552ed5f..38e480cb7 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -5,7 +5,6 @@ class DockerApiClient require 'socket' require 'json' - require_relative 'oms_common' require_relative 'omslog' require_relative 'DockerApiRestHelper' @@ -80,42 +79,50 @@ def listContainers() def getImageRepositoryImageTag(tagValue) result = ["", "", ""] - if !tagValue.empty? - # Find delimiters in the string of format repository/image:imagetag - slashLocation = tagValue.index('/') - colonLocation = tagValue.index(':') - if !colonLocation.nil? - if slashLocation.nil? - # image:imagetag - result[1] = tagValue[0..(colonLocation-1)] - else - # repository/image:imagetag - result[0] = tagValue[0..(slashLocation-1)] - result[1] = tagValue[(slashLocation + 1)..(colonLocation - 1)] + begin + if !tagValue.empty? + # Find delimiters in the string of format repository/image:imagetag + slashLocation = tagValue.index('/') + colonLocation = tagValue.index(':') + if !colonLocation.nil? + if slashLocation.nil? + # image:imagetag + result[1] = tagValue[0..(colonLocation-1)] + else + # repository/image:imagetag + result[0] = tagValue[0..(slashLocation-1)] + result[1] = tagValue[(slashLocation + 1)..(colonLocation - 1)] + end + result[2] = tagValue[(colonLocation + 1)..-1] end - result[2] = tagValue[(colonLocation + 1)..-1] end + rescue => errorStr + $log.warn("Exception at getImageRepositoryImageTag: #{errorStr} @ #{Time.now.utc.iso8601}") end return result end def getImageIdMap() result = nil - request = DockerApiRestHelper.restDockerImages - images = getResponse(request, true) - if !images.nil? && !images.empty? - result = {} - images.each do |image| - tagValue = "" - tags = image['RepoTags'] - if !tags.nil? && tags.kind_of?(Array) && tags.length > 0 - tagValue = tags[0] - end - idValue = image['Id'] - if !idValue.nil? - result[idValue] = getImageRepositoryImageTag(tagValue) + begin + request = DockerApiRestHelper.restDockerImages + images = getResponse(request, true) + if !images.nil? && !images.empty? + result = {} + images.each do |image| + tagValue = "" + tags = image['RepoTags'] + if !tags.nil? && tags.kind_of?(Array) && tags.length > 0 + tagValue = tags[0] + end + idValue = image['Id'] + if !idValue.nil? + result[idValue] = getImageRepositoryImageTag(tagValue) + end end end + rescue => errorStr + $log.warn("Exception at getImageIdMap: #{errorStr} @ #{Time.now.utc.iso8601}") end return result end From a5deb489023bdc221a20159042fdff3c6df08bf5 Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 26 Oct 2018 13:39:19 -0700 Subject: [PATCH 19/45] socket timeout --- source/code/plugin/DockerApiClient.rb | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index 38e480cb7..24bd286ef 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -5,11 +5,13 @@ class DockerApiClient require 'socket' require 'json' + require 'timeout' require_relative 'omslog' require_relative 'DockerApiRestHelper' @@SocketPath = "/var/run/docker.sock" @@ChunkSize = 4096 + @@TimeoutInSeconds = 5 def initialize end @@ -21,15 +23,24 @@ def getResponse(request, isMultiJson) begin socket = UNIXSocket.new(@@SocketPath) dockerResponse = "" + isTimeOut = false socket.write(request) # iterate through the response until the last chunk is less than the chunk size so that we can read all data in socket. loop do - responseChunk = socket.recv(@@ChunkSize) - dockerResponse += responseChunk + begin + responseChunk = "" + timeout(@@TimeoutInSeconds) do + responseChunk = socket.recv(@@ChunkSize) + end + dockerResponse += responseChunk + rescue Timeout::Error + $log.warn("Socket read timedout for request: #{request} @ #{Time.now.utc.iso8601}") + isTimeOut = true + end break if responseChunk.length < @@ChunkSize end socket.close - return parseResponse(dockerResponse, isMultiJson) + return (isTimeOut)? nil : parseResponse(dockerResponse, isMultiJson) rescue => errorStr $log.warn("Socket call failed for request: #{request} error: #{errorStr} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") end From cfb0a336120644615bf6a6fb48abf849c400c831 Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 26 Oct 2018 14:23:42 -0700 Subject: [PATCH 20/45] deleting test files --- source/code/plugin/socketClient.rb | 85 -------------------------- source/code/plugin/testdockerclient.rb | 6 -- 2 files changed, 91 deletions(-) delete mode 100644 source/code/plugin/socketClient.rb delete mode 100644 source/code/plugin/testdockerclient.rb diff --git a/source/code/plugin/socketClient.rb b/source/code/plugin/socketClient.rb deleted file mode 100644 index 9b76dc790..000000000 --- a/source/code/plugin/socketClient.rb +++ /dev/null @@ -1,85 +0,0 @@ -require "socket" -require 'json' - -socket = UNIXSocket.new('/var/run/docker.sock') - -socket.write("GET /info HTTP/1.1\r\nHost: localhost\r\n\r\n") -myresponse = "" -loop do -mystring = socket.recv(4096) -myresponse = myresponse + mystring -#puts "--------------------------------" -break if mystring.length < 4096 -end -puts myresponse -myjson = myresponse.match( /{.+}/ )[0] -myjson = "["+myjson+"]" -myparsedjson = JSON.parse(myjson) -puts "Hostname-" + myparsedjson[0]['Name'] - -#puts "==== Sending" -socket.write("GET /images/json?all=0 HTTP/1.1\r\nHost: localhost\r\n\r\n") - -#socket.write("GET /info HTTP/1.1\r\nHost: localhost\r\n\r\n") - -#socket.write("info HTTP/1.1\r\nHost: localhost\r\n\r\n") - -#puts "==== Getting Response" -#puts socket - -#puts socket.recvfrom(1024) - -#puts socket.gets - -#puts socket.recv.body - -myresponse = "" -loop do -mystring = socket.recv(4096) -myresponse = myresponse + mystring -#puts "--------------------------------" -break if mystring.length < 4096 -end - -#puts myresponse -#puts "-----------------------------------------------------" - -myjson = myresponse.match( /{.+}/ )[0] -myjson = "["+myjson+"]" -#puts myjson - - -myparsedjson = JSON.parse(myjson) -puts"---------------------------" -puts"IMAGES" -puts"---------------------------" -myparsedjson.each do |items| - puts items['Id'] -end -#while socket.recvfrom(1024) -# puts socket.recvfrom(1024) -#end - -#puts "here" -#puts "==== Sending" -socket.write("GET /containers/json?all=1 HTTP/1.1\r\nHost: localhost\r\n\r\n") -#puts "==== Getting Response" - -myresponse = "" -loop do -mystring = socket.recv(4096) -myresponse = myresponse + mystring -#puts "--------------------------------" -break if mystring.length < 4096 -end -myjson = myresponse.match( /{.+}/ )[0] -myjson = "["+myjson+"]" -myparsedjson = JSON.parse(myjson) -#puts myparsedjson -puts"---------------------------" -puts"CONTAINERS" -puts"---------------------------" -myparsedjson.each do |items| - puts items['Id'] -end -socket.close \ No newline at end of file diff --git a/source/code/plugin/testdockerclient.rb b/source/code/plugin/testdockerclient.rb deleted file mode 100644 index cadcad44d..000000000 --- a/source/code/plugin/testdockerclient.rb +++ /dev/null @@ -1,6 +0,0 @@ -require_relative 'DockerApiClient' - -myhost = DockerApiClient.getDockerHostName -puts myhost - -puts DockerApiClient.listContainers \ No newline at end of file From c2c2a0b189b7a43c0b491691b963da7cad715fbf Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 26 Oct 2018 14:26:26 -0700 Subject: [PATCH 21/45] adding log --- source/code/plugin/in_containerinventory.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 9d9c92def..4c1eb46cd 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -202,6 +202,7 @@ def enumerate eventStream.add(emitTime, wrapper) if wrapper end router.emit_stream(@tag, eventStream) if eventStream + $log.info("in_container_inventory::enumerate : Processing complete - emitted stream @ #{Time.now.utc.iso8601}") end rescue => errorStr $log.warn("Exception in enumerate container inventory: #{errorStr}") From fda2cc54f4c5e3d6234e43676c832d2400c0f651 Mon Sep 17 00:00:00 2001 From: rashmy Date: Fri, 26 Oct 2018 14:31:18 -0700 Subject: [PATCH 22/45] fixing comment --- source/code/plugin/in_containerinventory.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 4c1eb46cd..97d432016 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -60,7 +60,7 @@ def obtainContainerConfig(instance, container) instance['ComposeGroup'] = labelsValue['com.docker.compose.project'] end else - $log.warn("Attempt in ObtainContainerConfig to get container #{container['Id']} config information returned null") + $log.warn("Attempt in ObtainContainerConfig to get container: #{container['Id']} config information returned null") end rescue => errorStr $log.warn("Exception in obtainContainerConfig: #{errorStr}") @@ -75,7 +75,7 @@ def obtainContainerState(instance, container) # Exit codes less than 0 are not supported by the engine if exitCodeValue < 0 exitCodeValue = 128 - $log.info("obtainContainerState::Container #{container['Id']} returned negative exit code") + $log.info("obtainContainerState::Container: #{container['Id']} returned negative exit code") end instance['ExitCode'] = exitCodeValue if exitCodeValue > 0 From de76b9eb51e44292e9e7483e421d182761b46dd0 Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 29 Oct 2018 18:58:13 -0700 Subject: [PATCH 23/45] appinsights changes --- .../code/plugin/ApplicationInsightsUtility.rb | 70 +++++++++++++++++++ source/code/plugin/DockerApiClient.rb | 11 +++ source/code/plugin/DockerApiRestHelper.rb | 2 - source/code/plugin/in_containerinventory.rb | 3 + 4 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 source/code/plugin/ApplicationInsightsUtility.rb diff --git a/source/code/plugin/ApplicationInsightsUtility.rb b/source/code/plugin/ApplicationInsightsUtility.rb new file mode 100644 index 000000000..055239f35 --- /dev/null +++ b/source/code/plugin/ApplicationInsightsUtility.rb @@ -0,0 +1,70 @@ +#!/usr/local/bin/ruby +# frozen_string_literal: true + +class ApplicationInsightsUtility + require_relative 'application_insights' + require_relative 'omslog' + require 'json' + + @@HeartBeat = 'HeartBeatEvent' + @OmsAdminFilePath = '/etc/opt/microsoft/omsagent/conf/omsadmin.conf' + def initialize + tc = ApplicationInsights::TelemetryClient.new '9435b43f-97d5-4ded-8d90-b047958e6e87' + customProperties = {} + resourceInfo = ENV['AKS_RESOURCE_ID'] + if resourceInfo.nil? || resourceInfo.empty? + customProperties["ACSResourceName"] = ENV['ACS_RESOURCE_NAME'] + customProperties["ClusterType"] = 'ACS' + customProperties["SubscriptionID"] = "" + customProperties["ResourceGroupName"] = "" + customProperties["ClusterName"] = "" + customProperties["Region"] = "" + customProperties["AKS_RESOURCE_ID"] = "" + else + aksResourceId = ENV['AKS_RESOURCE_ID'] + customProperties["AKS_RESOURCE_ID"] = aksResourceId + begin + splitStrings = aksResourceId.split('/') + subscriptionId = splitStrings[2] + resourceGroupName = splitStrings[4] + clusterName = splitStrings[8] + rescue => errorStr + $log.warn("Exception in AppInsightsUtility: parsing AKS resourceId: #{aksResourceId}, error: #{errorStr}") + end + customProperties["ClusterType"] = 'AKS' + customProperties["SubscriptionID"] = subscriptionId + customProperties["ResourceGroupName"] = resourceGroupName + customProperties["ClusterName"] = clusterName + customProperties["Region"] = ENV['AKS_REGION'] + end + customProperties['ControllerType'] = 'DaemonSet' + end + + class << self + def sendHeartBeatEvent(pluginName, dockerInfo, hostName, workspaceId) + customProperties['WorkspaceID'] = workspaceId + eventName = pluginName + @@HeartBeat + customProperties['Computer'] = hostName + tc.track_event eventName , :properties => customProperties + end + + def sendCustomEvent(pluginName) + + end + + def sendExceptionLog(pluginName) + + end + + def getWorkspaceId() + adminConf = {} + f = File.open(@OmsAdminFilePath, "r") + f.each_line do |line| + splitStrings = line.split('=') + adminConf[splitStrings[0]] = splitStrings[1] + end + workspaceId = adminConf['WORKSPACE_ID'] + return workspaceId + end + end +end \ No newline at end of file diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index 24bd286ef..2963a1dc3 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -142,5 +142,16 @@ def dockerInspectContainer(id) request = DockerApiRestHelper.restDockerInspect(id) return getResponse(request, false) end + + def dockerInfo() + request = DockerApiRestHelper.restDockerVersion + response = getResponse(request, false) + dockerInfo = {} + if (response != nil) + dockerInfo['Version'] = response['Version'] + dockerInfo['ApiVersion'] = response['ApiVersion'] + end + return dockerInfo + end end end \ No newline at end of file diff --git a/source/code/plugin/DockerApiRestHelper.rb b/source/code/plugin/DockerApiRestHelper.rb index b9c9304eb..e10a6b294 100644 --- a/source/code/plugin/DockerApiRestHelper.rb +++ b/source/code/plugin/DockerApiRestHelper.rb @@ -44,5 +44,3 @@ def restDockerInfo() end end end - - \ No newline at end of file diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 97d432016..84241ba29 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -12,6 +12,7 @@ def initialize require_relative 'DockerApiClient' require_relative 'ContainerInventoryState' require_relative 'omslog' + require_relative 'application_insights' end config_param :run_interval, :time, :default => '1m' @@ -27,6 +28,8 @@ def start @condition = ConditionVariable.new @mutex = Mutex.new @thread = Thread.new(&method(:run_periodic)) + @dockerInfo = DockerApiClient.dockerInfo + @workspaceId = ApplicationInsightsUtility.getWorkspaceId end end From 6818c926f54556a5604e69eca4ca5ebc76590409 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 30 Oct 2018 11:41:50 -0700 Subject: [PATCH 24/45] changes --- .../code/plugin/ApplicationInsightsUtility.rb | 29 +++++++++++++++---- source/code/plugin/in_containerinventory.rb | 3 +- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/source/code/plugin/ApplicationInsightsUtility.rb b/source/code/plugin/ApplicationInsightsUtility.rb index 055239f35..e9c361b53 100644 --- a/source/code/plugin/ApplicationInsightsUtility.rb +++ b/source/code/plugin/ApplicationInsightsUtility.rb @@ -4,9 +4,11 @@ class ApplicationInsightsUtility require_relative 'application_insights' require_relative 'omslog' + require_relative 'DockerApiClient' require 'json' @@HeartBeat = 'HeartBeatEvent' + @@Exception = 'ExceptionEvent' @OmsAdminFilePath = '/etc/opt/microsoft/omsagent/conf/omsadmin.conf' def initialize tc = ApplicationInsights::TelemetryClient.new '9435b43f-97d5-4ded-8d90-b047958e6e87' @@ -38,22 +40,37 @@ def initialize customProperties["Region"] = ENV['AKS_REGION'] end customProperties['ControllerType'] = 'DaemonSet' + dockerInfo = DockerApiClient.dockerInfo + customProperties['DockerVersion'] = dockerInfo['Version'] + customProperties['DockerApiVersion'] = dockerInfo['ApiVersion'] + customProperties['WorkspaceID'] = getWorkspaceId end class << self - def sendHeartBeatEvent(pluginName, dockerInfo, hostName, workspaceId) - customProperties['WorkspaceID'] = workspaceId + def sendHeartBeatEvent(pluginName, properties) eventName = pluginName + @@HeartBeat - customProperties['Computer'] = hostName tc.track_event eventName , :properties => customProperties + tc.flush end - def sendCustomEvent(pluginName) - + def sendCustomEvent(pluginName, properties) + tc.track_metric 'LastProcessedContainerInventoryCount', properties['ContainerCount'], + :kind => ApplicationInsights::Channel::Contracts::DataPointType::MEASUREMENT, + :properties => { customProperties } + tc.flush end - def sendExceptionLog(pluginName) + def sendExceptionTelemetry(pluginName, errorStr) + eventName = pluginName + @@Exception + customProperties['Exception'] = errorStr + tc.track_event eventName , :properties => customProperties + tc.flush + end + def sendTelemetry(pluginName, properties) + customProperties['Computer'] = properties['Computer'] + sendHeartBeatEvent(properties) + sendCustomEvent(pluginName, properties) end def getWorkspaceId() diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 84241ba29..8d880e579 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -28,8 +28,7 @@ def start @condition = ConditionVariable.new @mutex = Mutex.new @thread = Thread.new(&method(:run_periodic)) - @dockerInfo = DockerApiClient.dockerInfo - @workspaceId = ApplicationInsightsUtility.getWorkspaceId + #@workspaceId = ApplicationInsightsUtility.getWorkspaceId end end From ddd2b88c54a608a36c03da9fbcf0b23101733cd3 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 30 Oct 2018 11:52:29 -0700 Subject: [PATCH 25/45] tel changes --- source/code/plugin/DockerApiClient.rb | 3 +++ source/code/plugin/in_containerinventory.rb | 10 ++++++++++ 2 files changed, 13 insertions(+) diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index 2963a1dc3..5128d12b3 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -8,6 +8,7 @@ class DockerApiClient require 'timeout' require_relative 'omslog' require_relative 'DockerApiRestHelper' + require_relative 'ApplicationInsightsUtility' @@SocketPath = "/var/run/docker.sock" @@ChunkSize = 4096 @@ -43,6 +44,7 @@ def getResponse(request, isMultiJson) return (isTimeOut)? nil : parseResponse(dockerResponse, isMultiJson) rescue => errorStr $log.warn("Socket call failed for request: #{request} error: #{errorStr} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") + ApplicationInsightsUtility.sendExceptionTelemetry('ContainerInventory', errorStr) end end @@ -61,6 +63,7 @@ def parseResponse(dockerResponse, isMultiJson) end rescue => errorStr $log.warn("Json parsing for docker response failed: #{errorStr} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") + ApplicationInsightsUtility.sendExceptionTelemetry('ContainerInventory', errorStr) end return parsedJsonResponse end diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 8d880e579..4cb579989 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -28,6 +28,7 @@ def start @condition = ConditionVariable.new @mutex = Mutex.new @thread = Thread.new(&method(:run_periodic)) + @@telemetryTimeTracker = DateTime.now.to_time.to_i #@workspaceId = ApplicationInsightsUtility.getWorkspaceId end end @@ -204,6 +205,15 @@ def enumerate eventStream.add(emitTime, wrapper) if wrapper end router.emit_stream(@tag, eventStream) if eventStream + timeDifference = (DateTime.now.to_time.to_i - @telemetryTimeTracker).abs + timeDifferenceInMinutes = timeDifference/60 + if (timeDifferenceInMinutes >= 5) + @telemetryTimeTracker = DateTime.now.to_time.to_i + telemetryProperties = {} + telemetryProperties['Computer'] = hostname + telemetryProperties['ContainerCount'] = containerInventory.length + ApplicationInsightsUtility.sendTelemetry('ContainerInventory') + end $log.info("in_container_inventory::enumerate : Processing complete - emitted stream @ #{Time.now.utc.iso8601}") end rescue => errorStr From b15eb56a6a24078cf614c1cb16e5350d5ee31d07 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 30 Oct 2018 12:21:35 -0700 Subject: [PATCH 26/45] changes --- .../code/plugin/ApplicationInsightsUtility.rb | 59 +++++++++---------- source/code/plugin/in_containerinventory.rb | 2 +- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/source/code/plugin/ApplicationInsightsUtility.rb b/source/code/plugin/ApplicationInsightsUtility.rb index e9c361b53..9311049b5 100644 --- a/source/code/plugin/ApplicationInsightsUtility.rb +++ b/source/code/plugin/ApplicationInsightsUtility.rb @@ -10,21 +10,21 @@ class ApplicationInsightsUtility @@HeartBeat = 'HeartBeatEvent' @@Exception = 'ExceptionEvent' @OmsAdminFilePath = '/etc/opt/microsoft/omsagent/conf/omsadmin.conf' + @@customProperties = {} + @@tc = ApplicationInsights::TelemetryClient.new '9435b43f-97d5-4ded-8d90-b047958e6e87' def initialize - tc = ApplicationInsights::TelemetryClient.new '9435b43f-97d5-4ded-8d90-b047958e6e87' - customProperties = {} resourceInfo = ENV['AKS_RESOURCE_ID'] if resourceInfo.nil? || resourceInfo.empty? - customProperties["ACSResourceName"] = ENV['ACS_RESOURCE_NAME'] - customProperties["ClusterType"] = 'ACS' - customProperties["SubscriptionID"] = "" - customProperties["ResourceGroupName"] = "" - customProperties["ClusterName"] = "" - customProperties["Region"] = "" - customProperties["AKS_RESOURCE_ID"] = "" + @@customProperties["ACSResourceName"] = ENV['ACS_RESOURCE_NAME'] + @@customProperties["ClusterType"] = 'ACS' + @@customProperties["SubscriptionID"] = "" + @@customProperties["ResourceGroupName"] = "" + @@customProperties["ClusterName"] = "" + @@customProperties["Region"] = "" + @@customProperties["AKS_RESOURCE_ID"] = "" else aksResourceId = ENV['AKS_RESOURCE_ID'] - customProperties["AKS_RESOURCE_ID"] = aksResourceId + @@customProperties["AKS_RESOURCE_ID"] = aksResourceId begin splitStrings = aksResourceId.split('/') subscriptionId = splitStrings[2] @@ -33,50 +33,49 @@ def initialize rescue => errorStr $log.warn("Exception in AppInsightsUtility: parsing AKS resourceId: #{aksResourceId}, error: #{errorStr}") end - customProperties["ClusterType"] = 'AKS' - customProperties["SubscriptionID"] = subscriptionId - customProperties["ResourceGroupName"] = resourceGroupName - customProperties["ClusterName"] = clusterName - customProperties["Region"] = ENV['AKS_REGION'] + @@customProperties["ClusterType"] = 'AKS' + @@customProperties["SubscriptionID"] = subscriptionId + @@customProperties["ResourceGroupName"] = resourceGroupName + @@customProperties["ClusterName"] = clusterName + @@customProperties["Region"] = ENV['AKS_REGION'] end - customProperties['ControllerType'] = 'DaemonSet' + @@customProperties['ControllerType'] = 'DaemonSet' dockerInfo = DockerApiClient.dockerInfo - customProperties['DockerVersion'] = dockerInfo['Version'] - customProperties['DockerApiVersion'] = dockerInfo['ApiVersion'] - customProperties['WorkspaceID'] = getWorkspaceId + @@customProperties['DockerVersion'] = dockerInfo['Version'] + @@customProperties['DockerApiVersion'] = dockerInfo['ApiVersion'] + @@customProperties['WorkspaceID'] = getWorkspaceId end class << self def sendHeartBeatEvent(pluginName, properties) eventName = pluginName + @@HeartBeat - tc.track_event eventName , :properties => customProperties - tc.flush + @@tc.track_event eventName , :properties => @@customProperties + @@tc.flush end def sendCustomEvent(pluginName, properties) - tc.track_metric 'LastProcessedContainerInventoryCount', properties['ContainerCount'], + @@tc.track_metric 'LastProcessedContainerInventoryCount', properties['ContainerCount'], :kind => ApplicationInsights::Channel::Contracts::DataPointType::MEASUREMENT, - :properties => { customProperties } - tc.flush + :properties => { @@customProperties } + @@tc.flush end def sendExceptionTelemetry(pluginName, errorStr) eventName = pluginName + @@Exception - customProperties['Exception'] = errorStr - tc.track_event eventName , :properties => customProperties - tc.flush + @@tc.track_exception errorStr , :properties => @@customProperties + @@tc.flush end def sendTelemetry(pluginName, properties) - customProperties['Computer'] = properties['Computer'] + @@customProperties['Computer'] = properties['Computer'] sendHeartBeatEvent(properties) sendCustomEvent(pluginName, properties) end def getWorkspaceId() adminConf = {} - f = File.open(@OmsAdminFilePath, "r") - f.each_line do |line| + confFile = File.open(@OmsAdminFilePath, "r") + confFile.each_line do |line| splitStrings = line.split('=') adminConf[splitStrings[0]] = splitStrings[1] end diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 4cb579989..8aaea4253 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -212,7 +212,7 @@ def enumerate telemetryProperties = {} telemetryProperties['Computer'] = hostname telemetryProperties['ContainerCount'] = containerInventory.length - ApplicationInsightsUtility.sendTelemetry('ContainerInventory') + ApplicationInsightsUtility.sendTelemetry('ContainerInventory', telemetryProperties) end $log.info("in_container_inventory::enumerate : Processing complete - emitted stream @ #{Time.now.utc.iso8601}") end From 9c279b7a704ab2880ee7d4c04598f9b51ca63276 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 30 Oct 2018 14:14:33 -0700 Subject: [PATCH 27/45] changes --- .../code/plugin/ApplicationInsightsUtility.rb | 61 +++++++++++-------- source/code/plugin/DockerApiClient.rb | 4 +- source/code/plugin/in_containerinventory.rb | 4 +- 3 files changed, 40 insertions(+), 29 deletions(-) diff --git a/source/code/plugin/ApplicationInsightsUtility.rb b/source/code/plugin/ApplicationInsightsUtility.rb index 9311049b5..2f4456f6a 100644 --- a/source/code/plugin/ApplicationInsightsUtility.rb +++ b/source/code/plugin/ApplicationInsightsUtility.rb @@ -13,18 +13,22 @@ class ApplicationInsightsUtility @@customProperties = {} @@tc = ApplicationInsights::TelemetryClient.new '9435b43f-97d5-4ded-8d90-b047958e6e87' def initialize - resourceInfo = ENV['AKS_RESOURCE_ID'] - if resourceInfo.nil? || resourceInfo.empty? - @@customProperties["ACSResourceName"] = ENV['ACS_RESOURCE_NAME'] - @@customProperties["ClusterType"] = 'ACS' - @@customProperties["SubscriptionID"] = "" - @@customProperties["ResourceGroupName"] = "" - @@customProperties["ClusterName"] = "" - @@customProperties["Region"] = "" - @@customProperties["AKS_RESOURCE_ID"] = "" - else - aksResourceId = ENV['AKS_RESOURCE_ID'] - @@customProperties["AKS_RESOURCE_ID"] = aksResourceId + end + + class << self + def initilizeutility() + resourceInfo = ENV['AKS_RESOURCE_ID'] + if resourceInfo.nil? || resourceInfo.empty? + @@customProperties["ACSResourceName"] = ENV['ACS_RESOURCE_NAME'] + @@customProperties["ClusterType"] = 'ACS' + @@customProperties["SubscriptionID"] = "" + @@customProperties["ResourceGroupName"] = "" + @@customProperties["ClusterName"] = "" + @@customProperties["Region"] = "" + @@customProperties["AKS_RESOURCE_ID"] = "" + else + aksResourceId = ENV['AKS_RESOURCE_ID'] + @@customProperties["AKS_RESOURCE_ID"] = aksResourceId begin splitStrings = aksResourceId.split('/') subscriptionId = splitStrings[2] @@ -33,20 +37,19 @@ def initialize rescue => errorStr $log.warn("Exception in AppInsightsUtility: parsing AKS resourceId: #{aksResourceId}, error: #{errorStr}") end - @@customProperties["ClusterType"] = 'AKS' - @@customProperties["SubscriptionID"] = subscriptionId - @@customProperties["ResourceGroupName"] = resourceGroupName - @@customProperties["ClusterName"] = clusterName - @@customProperties["Region"] = ENV['AKS_REGION'] + @@customProperties["ClusterType"] = 'AKS' + @@customProperties["SubscriptionID"] = subscriptionId + @@customProperties["ResourceGroupName"] = resourceGroupName + @@customProperties["ClusterName"] = clusterName + @@customProperties["Region"] = ENV['AKS_REGION'] + end + @@customProperties['ControllerType'] = 'DaemonSet' + dockerInfo = DockerApiClient.dockerInfo + @@customProperties['DockerVersion'] = dockerInfo['Version'] + @@customProperties['DockerApiVersion'] = dockerInfo['ApiVersion'] + @@customProperties['WorkspaceID'] = getWorkspaceId end - @@customProperties['ControllerType'] = 'DaemonSet' - dockerInfo = DockerApiClient.dockerInfo - @@customProperties['DockerVersion'] = dockerInfo['Version'] - @@customProperties['DockerApiVersion'] = dockerInfo['ApiVersion'] - @@customProperties['WorkspaceID'] = getWorkspaceId - end - class << self def sendHeartBeatEvent(pluginName, properties) eventName = pluginName + @@HeartBeat @@tc.track_event eventName , :properties => @@customProperties @@ -56,19 +59,25 @@ def sendHeartBeatEvent(pluginName, properties) def sendCustomEvent(pluginName, properties) @@tc.track_metric 'LastProcessedContainerInventoryCount', properties['ContainerCount'], :kind => ApplicationInsights::Channel::Contracts::DataPointType::MEASUREMENT, - :properties => { @@customProperties } + :properties => @@customProperties @@tc.flush end def sendExceptionTelemetry(pluginName, errorStr) + if @@customProperties.empty? || @@customProperties.nil? + initilizeutility + end eventName = pluginName + @@Exception @@tc.track_exception errorStr , :properties => @@customProperties @@tc.flush end def sendTelemetry(pluginName, properties) + if @@customProperties.empty? || @@customProperties.nil? + initilizeutility + end @@customProperties['Computer'] = properties['Computer'] - sendHeartBeatEvent(properties) + sendHeartBeatEvent(pluginName, properties) sendCustomEvent(pluginName, properties) end diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index 5128d12b3..e306550c9 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -147,13 +147,15 @@ def dockerInspectContainer(id) end def dockerInfo() - request = DockerApiRestHelper.restDockerVersion + request = DockerApiRestHelper.restDockerInfo response = getResponse(request, false) dockerInfo = {} + $log.warn("docker info returned before: #{response}") if (response != nil) dockerInfo['Version'] = response['Version'] dockerInfo['ApiVersion'] = response['ApiVersion'] end + $log.warn("docker info returned: #{dockerInfo}") return dockerInfo end end diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 8aaea4253..89dea5bd3 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -205,10 +205,10 @@ def enumerate eventStream.add(emitTime, wrapper) if wrapper end router.emit_stream(@tag, eventStream) if eventStream - timeDifference = (DateTime.now.to_time.to_i - @telemetryTimeTracker).abs + timeDifference = (DateTime.now.to_time.to_i - @@telemetryTimeTracker).abs timeDifferenceInMinutes = timeDifference/60 if (timeDifferenceInMinutes >= 5) - @telemetryTimeTracker = DateTime.now.to_time.to_i + @@telemetryTimeTracker = DateTime.now.to_time.to_i telemetryProperties = {} telemetryProperties['Computer'] = hostname telemetryProperties['ContainerCount'] = containerInventory.length From d14324fc2d3b29ff7570534e0e3d32c8f9f33222 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 30 Oct 2018 14:16:00 -0700 Subject: [PATCH 28/45] changes --- source/code/plugin/DockerApiClient.rb | 2 +- source/code/plugin/DockerApiRestHelper.rb | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index e306550c9..09ecbb243 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -147,7 +147,7 @@ def dockerInspectContainer(id) end def dockerInfo() - request = DockerApiRestHelper.restDockerInfo + request = DockerApiRestHelper.restDockerVersion response = getResponse(request, false) dockerInfo = {} $log.warn("docker info returned before: #{response}") diff --git a/source/code/plugin/DockerApiRestHelper.rb b/source/code/plugin/DockerApiRestHelper.rb index e10a6b294..76361b122 100644 --- a/source/code/plugin/DockerApiRestHelper.rb +++ b/source/code/plugin/DockerApiRestHelper.rb @@ -42,5 +42,14 @@ def restDockerInfo() return "GET /info HTTP/1.1\r\nHost: localhost\r\n\r\n"; end end + + # Create the REST request to get docker info + # https://docs.docker.com/engine/api/v1.21/#21-containers + # returns Request in string format + def restDockerVersion() + begin + return "GET /version HTTP/1.1\r\nHost: localhost\r\n\r\n"; + end + end end -end +end \ No newline at end of file From 9b5e0e7006b4e4e05542d314f346735cc3b6687d Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 30 Oct 2018 16:59:22 -0700 Subject: [PATCH 29/45] changes --- installer/conf/container.conf | 4 +- .../code/plugin/ApplicationInsightsUtility.rb | 144 +++++++++++------- source/code/plugin/ContainerInventoryState.rb | 4 + source/code/plugin/DockerApiClient.rb | 8 +- source/code/plugin/in_containerinventory.rb | 19 ++- 5 files changed, 112 insertions(+), 67 deletions(-) diff --git a/installer/conf/container.conf b/installer/conf/container.conf index cc01244c9..84be0d05f 100755 --- a/installer/conf/container.conf +++ b/installer/conf/container.conf @@ -16,7 +16,7 @@ # Container inventory type containerinventory - tag oms.containerinsights.ContainerInventory + tag oms.containerinsights.containerinventory run_interval 60s log_level debug @@ -52,7 +52,7 @@ max_retry_wait 9m - + type out_oms log_level debug num_threads 5 diff --git a/source/code/plugin/ApplicationInsightsUtility.rb b/source/code/plugin/ApplicationInsightsUtility.rb index 2f4456f6a..b56519f2a 100644 --- a/source/code/plugin/ApplicationInsightsUtility.rb +++ b/source/code/plugin/ApplicationInsightsUtility.rb @@ -6,90 +6,126 @@ class ApplicationInsightsUtility require_relative 'omslog' require_relative 'DockerApiClient' require 'json' + require "base64" @@HeartBeat = 'HeartBeatEvent' @@Exception = 'ExceptionEvent' + @@AcsClusterType = 'ACS' + @@AksClusterType = 'AKS' + @@DaemonsetControllerType = 'DaemonSet' @OmsAdminFilePath = '/etc/opt/microsoft/omsagent/conf/omsadmin.conf' - @@customProperties = {} - @@tc = ApplicationInsights::TelemetryClient.new '9435b43f-97d5-4ded-8d90-b047958e6e87' + @@EnvAcsResourceName = 'ACS_RESOURCE_NAME' + @@EnvAksRegion = 'AKS_REGION' + @@EnvAgentVersion = 'AgentVersion' + @@CustomProperties = {} + encodedAppInsightsKey = ENV['APPLICATIONINSIGHTS_AUTH'] + decodedAppInsightsKey = Base64.decode64(encodedAppInsightsKey) + @@Tc = ApplicationInsights::TelemetryClient.new decodedAppInsightsKey + def initialize end class << self def initilizeutility() - resourceInfo = ENV['AKS_RESOURCE_ID'] - if resourceInfo.nil? || resourceInfo.empty? - @@customProperties["ACSResourceName"] = ENV['ACS_RESOURCE_NAME'] - @@customProperties["ClusterType"] = 'ACS' - @@customProperties["SubscriptionID"] = "" - @@customProperties["ResourceGroupName"] = "" - @@customProperties["ClusterName"] = "" - @@customProperties["Region"] = "" - @@customProperties["AKS_RESOURCE_ID"] = "" - else - aksResourceId = ENV['AKS_RESOURCE_ID'] - @@customProperties["AKS_RESOURCE_ID"] = aksResourceId begin - splitStrings = aksResourceId.split('/') - subscriptionId = splitStrings[2] - resourceGroupName = splitStrings[4] - clusterName = splitStrings[8] + resourceInfo = ENV['AKS_RESOURCE_ID'] + if resourceInfo.nil? || resourceInfo.empty? + @@CustomProperties["ACSResourceName"] = ENV[@@EnvAcsResourceName] + @@CustomProperties["ClusterType"] = @@AcsClusterType + @@CustomProperties["SubscriptionID"] = "" + @@CustomProperties["ResourceGroupName"] = "" + @@CustomProperties["ClusterName"] = "" + @@CustomProperties["Region"] = "" + @@CustomProperties["AKS_RESOURCE_ID"] = "" + else + aksResourceId = ENV['AKS_RESOURCE_ID'] + @@CustomProperties["AKS_RESOURCE_ID"] = aksResourceId + @@CustomProperties["ACSResourceName"] = "" + begin + splitStrings = aksResourceId.split('/') + subscriptionId = splitStrings[2] + resourceGroupName = splitStrings[4] + clusterName = splitStrings[8] + rescue => errorStr + $log.warn("Exception in AppInsightsUtility: parsing AKS resourceId: #{aksResourceId}, error: #{errorStr}") + end + @@CustomProperties["ClusterType"] = @@AksClusterType + @@CustomProperties["SubscriptionID"] = subscriptionId + @@CustomProperties["ResourceGroupName"] = resourceGroupName + @@CustomProperties["ClusterName"] = clusterName + @@CustomProperties["Region"] = ENV[@@EnvAksRegion] + end + @@CustomProperties['ControllerType'] = @@DaemonsetControllerType + dockerInfo = DockerApiClient.dockerInfo + @@CustomProperties['DockerVersion'] = dockerInfo['Version'] + @@CustomProperties['DockerApiVersion'] = dockerInfo['ApiVersion'] + @@CustomProperties['WorkspaceID'] = getWorkspaceId + @@CustomProperties['AgentVersion'] = ENV[@@EnvAgentVersion] rescue => errorStr - $log.warn("Exception in AppInsightsUtility: parsing AKS resourceId: #{aksResourceId}, error: #{errorStr}") - end - @@customProperties["ClusterType"] = 'AKS' - @@customProperties["SubscriptionID"] = subscriptionId - @@customProperties["ResourceGroupName"] = resourceGroupName - @@customProperties["ClusterName"] = clusterName - @@customProperties["Region"] = ENV['AKS_REGION'] + $log.warn("Exception in AppInsightsUtility: initilizeUtility - error: #{errorStr}") end - @@customProperties['ControllerType'] = 'DaemonSet' - dockerInfo = DockerApiClient.dockerInfo - @@customProperties['DockerVersion'] = dockerInfo['Version'] - @@customProperties['DockerApiVersion'] = dockerInfo['ApiVersion'] - @@customProperties['WorkspaceID'] = getWorkspaceId end def sendHeartBeatEvent(pluginName, properties) - eventName = pluginName + @@HeartBeat - @@tc.track_event eventName , :properties => @@customProperties - @@tc.flush + begin + eventName = pluginName + @@HeartBeat + @@tc.track_event eventName , :properties => @@CustomProperties + @@tc.flush + rescue =>errorStr + $log.warn("Exception in AppInsightsUtility: sendHeartBeatEvent - error: #{errorStr}") + end end def sendCustomEvent(pluginName, properties) - @@tc.track_metric 'LastProcessedContainerInventoryCount', properties['ContainerCount'], - :kind => ApplicationInsights::Channel::Contracts::DataPointType::MEASUREMENT, - :properties => @@customProperties - @@tc.flush + begin + @@tc.track_metric 'LastProcessedContainerInventoryCount', properties['ContainerCount'], + :kind => ApplicationInsights::Channel::Contracts::DataPointType::MEASUREMENT, + :properties => @@CustomProperties + @@tc.flush + rescue => errorStr + $log.warn("Exception in AppInsightsUtility: sendCustomEvent - error: #{errorStr}") + end end def sendExceptionTelemetry(pluginName, errorStr) - if @@customProperties.empty? || @@customProperties.nil? - initilizeutility + begin + if @@CustomProperties.empty? || @@CustomProperties.nil? + initilizeutility + end + eventName = pluginName + @@Exception + @@tc.track_exception errorStr , :properties => @@CustomProperties + @@tc.flush + rescue => errorStr + $log.warn("Exception in AppInsightsUtility: sendExceptionTelemetry - error: #{errorStr}") end - eventName = pluginName + @@Exception - @@tc.track_exception errorStr , :properties => @@customProperties - @@tc.flush end def sendTelemetry(pluginName, properties) - if @@customProperties.empty? || @@customProperties.nil? - initilizeutility + begin + if @@CustomProperties.empty? || @@CustomProperties.nil? + initilizeutility + end + @@CustomProperties['Computer'] = properties['Computer'] + sendHeartBeatEvent(pluginName, properties) + sendCustomEvent(pluginName, properties) + rescue + $log.warn("Exception in AppInsightsUtility: sendTelemetry - error: #{errorStr}") end - @@customProperties['Computer'] = properties['Computer'] - sendHeartBeatEvent(pluginName, properties) - sendCustomEvent(pluginName, properties) end def getWorkspaceId() - adminConf = {} - confFile = File.open(@OmsAdminFilePath, "r") - confFile.each_line do |line| - splitStrings = line.split('=') - adminConf[splitStrings[0]] = splitStrings[1] + begin + adminConf = {} + confFile = File.open(@OmsAdminFilePath, "r") + confFile.each_line do |line| + splitStrings = line.split('=') + adminConf[splitStrings[0]] = splitStrings[1] + end + workspaceId = adminConf['WORKSPACE_ID'] + return workspaceId + rescue => errorStr + $log.warn("Exception in AppInsightsUtility: getWorkspaceId - error: #{errorStr}") end - workspaceId = adminConf['WORKSPACE_ID'] - return workspaceId end end end \ No newline at end of file diff --git a/source/code/plugin/ContainerInventoryState.rb b/source/code/plugin/ContainerInventoryState.rb index 8eaf47e1d..7e5ca18e8 100644 --- a/source/code/plugin/ContainerInventoryState.rb +++ b/source/code/plugin/ContainerInventoryState.rb @@ -10,6 +10,7 @@ def initialize end class << self + # Write the container information to disk with the data that is obtained from the current plugin execution def writeContainerState(container) containerId = container['InstanceID'] if !containerId.nil? && !containerId.empty? @@ -27,6 +28,7 @@ def writeContainerState(container) end end + # Reads the container state for the deleted container def readContainerState(containerId) begin containerObject = nil @@ -47,6 +49,8 @@ def readContainerState(containerId) return containerObject end + # Gets the containers that were written to the disk with the previous plugin invocation but do not exist in the current container list + # Doing this because we need to update the container state to deleted. Else this will stay running forever. def getDeletedContainers(containerIds) deletedContainers = nil begin diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index 09ecbb243..2f03f30c8 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -18,9 +18,8 @@ def initialize end class << self - # Make docker socket call to + # Make docker socket call for requests def getResponse(request, isMultiJson) - #TODO: Add retries begin socket = UNIXSocket.new(@@SocketPath) dockerResponse = "" @@ -91,6 +90,7 @@ def listContainers() return ids end + # This method splits the tag value into an array - repository, image and tag def getImageRepositoryImageTag(tagValue) result = ["", "", ""] begin @@ -116,6 +116,7 @@ def getImageRepositoryImageTag(tagValue) return result end + # Image is in the format repository/image:imagetag - This method creates a hash of image id and repository, image and tag def getImageIdMap() result = nil begin @@ -146,16 +147,15 @@ def dockerInspectContainer(id) return getResponse(request, false) end + # This method returns docker version and docker api version for telemetry def dockerInfo() request = DockerApiRestHelper.restDockerVersion response = getResponse(request, false) dockerInfo = {} - $log.warn("docker info returned before: #{response}") if (response != nil) dockerInfo['Version'] = response['Version'] dockerInfo['ApiVersion'] = response['ApiVersion'] end - $log.warn("docker info returned: #{dockerInfo}") return dockerInfo end end diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 89dea5bd3..7072d3c2a 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -6,6 +6,12 @@ module Fluent class Container_Inventory_Input < Input Plugin.register_input('containerinventory', self) + @@PluginName = 'ContainerInventory' + @@RunningState = 'Running' + @@FailedState = 'Failed' + @@StoppedState = 'Stopped' + @@PausedState = 'Paused' + def initialize super require 'json' @@ -16,7 +22,7 @@ def initialize end config_param :run_interval, :time, :default => '1m' - config_param :tag, :string, :default => "oms.containerinsights.ContainerInventory" + config_param :tag, :string, :default => "oms.containerinsights.containerinventory" def configure (conf) super @@ -29,7 +35,6 @@ def start @mutex = Mutex.new @thread = Thread.new(&method(:run_periodic)) @@telemetryTimeTracker = DateTime.now.to_time.to_i - #@workspaceId = ApplicationInsightsUtility.getWorkspaceId end end @@ -82,7 +87,7 @@ def obtainContainerState(instance, container) end instance['ExitCode'] = exitCodeValue if exitCodeValue > 0 - instance['State'] = "Failed" + instance['State'] = @@FailedState else # Set the Container status : Running/Paused/Stopped runningValue = stateValue['Running'] @@ -90,12 +95,12 @@ def obtainContainerState(instance, container) pausedValue = stateValue['Paused'] # Checking for paused within running is true state because docker returns true for both Running and Paused fields when the container is paused if pausedValue - instance['State'] = "Paused" + instance['State'] = @@PausedState else - instance['State'] = "Running" + instance['State'] = @@RunningState end else - instance['State'] = "Stopped" + instance['State'] = @@StoppedState end end instance['StartedTime'] = stateValue['StartedAt'] @@ -212,7 +217,7 @@ def enumerate telemetryProperties = {} telemetryProperties['Computer'] = hostname telemetryProperties['ContainerCount'] = containerInventory.length - ApplicationInsightsUtility.sendTelemetry('ContainerInventory', telemetryProperties) + ApplicationInsightsUtility.sendTelemetry(@@PluginName, telemetryProperties) end $log.info("in_container_inventory::enumerate : Processing complete - emitted stream @ #{Time.now.utc.iso8601}") end From 17c308ee01b19d63a5a02fe67b5b947496325d84 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 30 Oct 2018 17:03:47 -0700 Subject: [PATCH 30/45] lib changes --- .../code/plugin/ApplicationInsightsUtility.rb | 2 +- .../code/plugin/lib/application_insights.rb | 9 + .../channel/asynchronous_queue.rb | 58 +++++ .../channel/asynchronous_sender.rb | 133 ++++++++++ .../channel/contracts/application.rb | 13 + .../channel/contracts/availability_data.rb | 34 +++ .../channel/contracts/base.rb | 13 + .../channel/contracts/cloud.rb | 14 ++ .../channel/contracts/data.rb | 14 ++ .../channel/contracts/data_point.rb | 25 ++ .../channel/contracts/data_point_type.rb | 7 + .../channel/contracts/dependency_kind.rb | 9 + .../contracts/dependency_source_type.rb | 9 + .../channel/contracts/device.rb | 18 ++ .../channel/contracts/domain.rb | 10 + .../channel/contracts/envelope.rb | 32 +++ .../channel/contracts/event_data.rb | 28 +++ .../channel/contracts/exception_data.rb | 35 +++ .../channel/contracts/exception_details.rb | 28 +++ .../channel/contracts/internal.rb | 15 ++ .../channel/contracts/json_serializable.rb | 59 +++++ .../channel/contracts/location.rb | 13 + .../channel/contracts/message_data.rb | 24 ++ .../channel/contracts/metric_data.rb | 27 ++ .../channel/contracts/operation.rb | 17 ++ .../channel/contracts/page_view_data.rb | 33 +++ .../channel/contracts/page_view_perf_data.rb | 39 +++ .../contracts/remote_dependency_data.rb | 40 +++ .../channel/contracts/reopenings.rb | 27 ++ .../channel/contracts/request_data.rb | 35 +++ .../channel/contracts/session.rb | 14 ++ .../channel/contracts/severity_level.rb | 13 + .../channel/contracts/stack_frame.rb | 17 ++ .../channel/contracts/user.rb | 15 ++ .../lib/application_insights/channel/event.rb | 68 +++++ .../channel/queue_base.rb | 73 ++++++ .../channel/sender_base.rb | 88 +++++++ .../channel/synchronous_queue.rb | 45 ++++ .../channel/synchronous_sender.rb | 17 ++ .../channel/telemetry_channel.rb | 131 ++++++++++ .../channel/telemetry_context.rb | 85 +++++++ .../rack/track_request.rb | 154 ++++++++++++ .../application_insights/telemetry_client.rb | 232 ++++++++++++++++++ .../unhandled_exception.rb | 49 ++++ .../lib/application_insights/version.rb | 3 + 45 files changed, 1823 insertions(+), 1 deletion(-) create mode 100644 source/code/plugin/lib/application_insights.rb create mode 100644 source/code/plugin/lib/application_insights/channel/asynchronous_queue.rb create mode 100644 source/code/plugin/lib/application_insights/channel/asynchronous_sender.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/application.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/availability_data.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/base.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/cloud.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/data.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/data_point.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/data_point_type.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/dependency_kind.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/dependency_source_type.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/device.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/domain.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/envelope.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/event_data.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/exception_data.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/exception_details.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/internal.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/json_serializable.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/location.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/message_data.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/metric_data.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/operation.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/page_view_data.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/page_view_perf_data.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/remote_dependency_data.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/reopenings.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/request_data.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/session.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/severity_level.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/stack_frame.rb create mode 100644 source/code/plugin/lib/application_insights/channel/contracts/user.rb create mode 100644 source/code/plugin/lib/application_insights/channel/event.rb create mode 100644 source/code/plugin/lib/application_insights/channel/queue_base.rb create mode 100644 source/code/plugin/lib/application_insights/channel/sender_base.rb create mode 100644 source/code/plugin/lib/application_insights/channel/synchronous_queue.rb create mode 100644 source/code/plugin/lib/application_insights/channel/synchronous_sender.rb create mode 100644 source/code/plugin/lib/application_insights/channel/telemetry_channel.rb create mode 100644 source/code/plugin/lib/application_insights/channel/telemetry_context.rb create mode 100644 source/code/plugin/lib/application_insights/rack/track_request.rb create mode 100644 source/code/plugin/lib/application_insights/telemetry_client.rb create mode 100644 source/code/plugin/lib/application_insights/unhandled_exception.rb create mode 100644 source/code/plugin/lib/application_insights/version.rb diff --git a/source/code/plugin/ApplicationInsightsUtility.rb b/source/code/plugin/ApplicationInsightsUtility.rb index b56519f2a..e25b7fa23 100644 --- a/source/code/plugin/ApplicationInsightsUtility.rb +++ b/source/code/plugin/ApplicationInsightsUtility.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true class ApplicationInsightsUtility - require_relative 'application_insights' + require_relative 'lib/application_insights' require_relative 'omslog' require_relative 'DockerApiClient' require 'json' diff --git a/source/code/plugin/lib/application_insights.rb b/source/code/plugin/lib/application_insights.rb new file mode 100644 index 000000000..0a683d484 --- /dev/null +++ b/source/code/plugin/lib/application_insights.rb @@ -0,0 +1,9 @@ +require_relative 'application_insights/telemetry_client' +require_relative 'application_insights/unhandled_exception' +require_relative 'application_insights/version' + +module ApplicationInsights + module Rack + autoload :TrackRequest, "application_insights/rack/track_request" + end +end diff --git a/source/code/plugin/lib/application_insights/channel/asynchronous_queue.rb b/source/code/plugin/lib/application_insights/channel/asynchronous_queue.rb new file mode 100644 index 000000000..333f6968b --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/asynchronous_queue.rb @@ -0,0 +1,58 @@ +require_relative 'event' +require_relative 'queue_base' + +module ApplicationInsights + module Channel + # An asynchronous queue for use in conjunction with the {AsynchronousSender}. + # The queue will notify the sender that it needs to pick up items when it + # reaches {#max_queue_length}, or when the consumer calls {#flush} via the + # {#flush_notification} event. + # + # @example + # require 'application_insights' + # require 'thread' + # queue = ApplicationInsights::Channel::AsynchronousQueue.new nil + # Thread.new do + # sleep 1 + # queue.push 1 + # queue.flush + # end + # queue.flush_notification.wait + # queue.flush_notification.clear + # result = queue.pop + class AsynchronousQueue < QueueBase + # Initializes a new instance of the class. + # @param [SenderBase] sender the sender object that will be used in + # conjunction with this queue. In addition to the sender object must + # support a {AsynchronousSender#start} method which is invoked each time + # an item is pushed to the queue as well as use the {#flush_notification} + # event. + def initialize(sender) + @flush_notification = Event.new + super sender + end + + # The flush notification {ApplicationInsights::Channel::Event} that the {#sender} + # will use to get notified that a flush is needed. + # @return [Event] object that the {#sender} can wait on. + attr_reader :flush_notification + + # Adds the passed in item object to the queue and notifies the {#sender} + # to start an asynchronous send operation + # by calling {AsynchronousSender#start}. + # @param [Contracts::Envelope] item the telemetry envelope object to send + # to the service. + def push(item) + super item + @sender.start if @sender + end + + # Flushes the current queue by notifying the {#sender} via the + # {#flush_notification} event. + def flush + @flush_notification.set + @sender.start if @sender + end + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/asynchronous_sender.rb b/source/code/plugin/lib/application_insights/channel/asynchronous_sender.rb new file mode 100644 index 000000000..da573f08c --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/asynchronous_sender.rb @@ -0,0 +1,133 @@ +require_relative 'sender_base' +require 'thread' + +module ApplicationInsights + module Channel + # An asynchronous sender that works in conjunction with the {AsynchronousQueue}. + # The sender object will start a worker thread that will pull items from the + # {#queue}. The thread will be created when the client calls {#start} and + # will check for queue items every {#send_interval} seconds. The worker thread + # can also be forced to check the queue by setting the + # {AsynchronousQueue#flush_notification} event. + # + # - If no items are found, the thread will go back to sleep. + # - If items are found, the worker thread will send items to the specified + # service in batches of {#send_buffer_size}. + # + # If no queue items are found for {#send_time} seconds, the worker thread + # will shut down (and {#start} will need to be called again). + class AsynchronousSender < SenderBase + SERVICE_ENDPOINT_URI = 'https://dc.services.visualstudio.com/v2/track' + # Initializes a new instance of the class. + # @param [String] service_endpoint_uri the address of the service to send + # telemetry data to. + def initialize(service_endpoint_uri = SERVICE_ENDPOINT_URI) + @send_interval = 1.0 + @send_remaining_time = 0 + @send_time = 3.0 + @lock_work_thread = Mutex.new + @work_thread = nil + @start_notification_processed = true + super service_endpoint_uri + end + + # The time span in seconds at which the the worker thread will check the + # {#queue} for items (defaults to: 1.0). + # @return [Fixnum] the interval in seconds. + attr_accessor :send_interval + + # The time span in seconds for which the worker thread will stay alive if + # no items are found in the {#queue} (defaults to 3.0). + # @return [Fixnum] the interval in seconds. + attr_accessor :send_time + + # The worker thread which checks queue items and send data every + # (#send_interval) seconds or upon flush. + # @return [Thread] the work thread + attr_reader :work_thread + + # Calling this method will create a worker thread that checks the {#queue} + # every {#send_interval} seconds for a total duration of {#send_time} + # seconds for new items. If a worker thread has already been created, + # calling this method does nothing. + def start + @start_notification_processed = false + # Maintain one working thread at one time + unless @work_thread + @lock_work_thread.synchronize do + unless @work_thread + local_send_interval = [@send_interval, 0.1].max + @send_remaining_time = [@send_time, local_send_interval].max + @work_thread = Thread.new { run } + @work_thread.abort_on_exception = false + end + end + end + end + + private + + def run + # save the queue locally + local_queue = @queue + if local_queue.nil? + @work_thread = nil + return + end + + begin + # fix up the send interval (can't be lower than 100ms) + local_send_interval = [@send_interval, 0.1].max + + while true + @start_notification_processed = true + while true + # get at most @send_buffer_size items from the queue + data = [] + @send_buffer_size.downto(1) do + item = local_queue.pop + break if not item + data.push item + end + + # if we didn't get any items from the queue, we're done here + break if data.length == 0 + + # reset the send time + @send_remaining_time = @send_time + + # finally send the data + send data + end + + # wait at most @send_interval ms (or until we get signalled) + result = local_queue.flush_notification.wait local_send_interval + if result + local_queue.flush_notification.clear + next + end + + # decrement the remaining time + @send_remaining_time -= local_send_interval + # If remaining time <=0 and there is no start notification unprocessed, + # then stop the working thread + if @send_remaining_time <= 0 && @start_notification_processed + # Note: there is still a chance some start notification could be + # missed, e.g., the start method got triggered between the above and + # following line. However the data is not lost as it would be + # processed later when next start notification comes after the worker + # thread stops. The cost to ensure no notification miss is high where + # a lock is required each time the start method calls. + @work_thread = nil + break + end + end + rescue Exception => e + # Make sure work_thread sets to nil when it terminates abnormally + @work_thread = nil + @logger.error('application_insights') { "Asynchronous sender work thread terminated abnormally: #{e.to_s}" } + end + end + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/application.rb b/source/code/plugin/lib/application_insights/channel/contracts/application.rb new file mode 100644 index 000000000..071c37385 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/application.rb @@ -0,0 +1,13 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class Application + include JsonSerializable + + attr_accessor :ver + + attribute_mapping( + ver: 'ai.application.ver' + ) + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/availability_data.rb b/source/code/plugin/lib/application_insights/channel/contracts/availability_data.rb new file mode 100644 index 000000000..d560dd15b --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/availability_data.rb @@ -0,0 +1,34 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class AvailabilityData + include JsonSerializable + + attr_accessor :ver, :id, :name, :duration, :success, :run_location, :message, + :properties, :measurements + + attribute_mapping( + ver: 'ver', + id: 'id', + name: 'name', + duration: 'duration', + success: 'success', + run_location: 'runLocation', + message: 'message', + properties: 'properties', + measurements: 'measurements' + ) + + def ver + @ver ||= 2 + end + + def properties + @properties ||= {} + end + + def measurements + @measurements ||= {} + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/base.rb b/source/code/plugin/lib/application_insights/channel/contracts/base.rb new file mode 100644 index 000000000..bb88a4625 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/base.rb @@ -0,0 +1,13 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class Base + include JsonSerializable + + attr_accessor :base_type + + attribute_mapping( + base_type: 'baseType' + ) + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/cloud.rb b/source/code/plugin/lib/application_insights/channel/contracts/cloud.rb new file mode 100644 index 000000000..5aaeeee04 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/cloud.rb @@ -0,0 +1,14 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class Cloud + include JsonSerializable + + attr_accessor :role, :role_instance + + attribute_mapping( + role: 'ai.cloud.role', + role_instance: 'ai.cloud.roleInstance' + ) + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/data.rb b/source/code/plugin/lib/application_insights/channel/contracts/data.rb new file mode 100644 index 000000000..c7184edfd --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/data.rb @@ -0,0 +1,14 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class Data + include JsonSerializable + + attr_accessor :base_type, :base_data + + attribute_mapping( + base_type: 'baseType', + base_data: 'baseData' + ) + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/data_point.rb b/source/code/plugin/lib/application_insights/channel/contracts/data_point.rb new file mode 100644 index 000000000..6556b351b --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/data_point.rb @@ -0,0 +1,25 @@ +require_relative 'json_serializable' +require_relative 'data_point_type' + +module ApplicationInsights::Channel::Contracts + class DataPoint + include JsonSerializable + + attr_accessor :ns, :name, :kind, :value, :count, :min, :max, :std_dev + + attribute_mapping( + ns: 'ns', + name: 'name', + kind: 'kind', + value: 'value', + count: 'count', + min: 'min', + max: 'max', + std_dev: 'stdDev' + ) + + def kind + @kind ||= DataPointType::MEASUREMENT + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/data_point_type.rb b/source/code/plugin/lib/application_insights/channel/contracts/data_point_type.rb new file mode 100644 index 000000000..f9816e4a9 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/data_point_type.rb @@ -0,0 +1,7 @@ +module ApplicationInsights::Channel::Contracts + class DataPointType + MEASUREMENT = 0 + + AGGREGATION = 1 + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/dependency_kind.rb b/source/code/plugin/lib/application_insights/channel/contracts/dependency_kind.rb new file mode 100644 index 000000000..38a441499 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/dependency_kind.rb @@ -0,0 +1,9 @@ +module ApplicationInsights::Channel::Contracts + class DependencyKind + SQL = 0 + + HTTP = 1 + + OTHER = 2 + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/dependency_source_type.rb b/source/code/plugin/lib/application_insights/channel/contracts/dependency_source_type.rb new file mode 100644 index 000000000..a68dad72b --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/dependency_source_type.rb @@ -0,0 +1,9 @@ +module ApplicationInsights::Channel::Contracts + class DependencySourceType + UNDEFINED = 0 + + AIC = 1 + + APMC = 2 + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/device.rb b/source/code/plugin/lib/application_insights/channel/contracts/device.rb new file mode 100644 index 000000000..af6855102 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/device.rb @@ -0,0 +1,18 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class Device + include JsonSerializable + + attr_accessor :id, :locale, :model, :oem_name, :os_version, :type + + attribute_mapping( + id: 'ai.device.id', + locale: 'ai.device.locale', + model: 'ai.device.model', + oem_name: 'ai.device.oemName', + os_version: 'ai.device.osVersion', + type: 'ai.device.type' + ) + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/domain.rb b/source/code/plugin/lib/application_insights/channel/contracts/domain.rb new file mode 100644 index 000000000..8a7ba880d --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/domain.rb @@ -0,0 +1,10 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class Domain + include JsonSerializable + + attribute_mapping( + ) + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/envelope.rb b/source/code/plugin/lib/application_insights/channel/contracts/envelope.rb new file mode 100644 index 000000000..b8608e388 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/envelope.rb @@ -0,0 +1,32 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class Envelope + include JsonSerializable + + attr_accessor :ver, :name, :time, :sample_rate, :seq, :i_key, :tags, :data + + attribute_mapping( + ver: 'ver', + name: 'name', + time: 'time', + sample_rate: 'sampleRate', + seq: 'seq', + i_key: 'iKey', + tags: 'tags', + data: 'data' + ) + + def ver + @ver ||= 1 + end + + def sample_rate + @sample_rate ||= 100.0 + end + + def tags + @tags ||= {} + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/event_data.rb b/source/code/plugin/lib/application_insights/channel/contracts/event_data.rb new file mode 100644 index 000000000..4bfb16124 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/event_data.rb @@ -0,0 +1,28 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class EventData + include JsonSerializable + + attr_accessor :ver, :name, :properties, :measurements + + attribute_mapping( + ver: 'ver', + name: 'name', + properties: 'properties', + measurements: 'measurements' + ) + + def ver + @ver ||= 2 + end + + def properties + @properties ||= {} + end + + def measurements + @measurements ||= {} + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/exception_data.rb b/source/code/plugin/lib/application_insights/channel/contracts/exception_data.rb new file mode 100644 index 000000000..5cffd1253 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/exception_data.rb @@ -0,0 +1,35 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class ExceptionData + include JsonSerializable + + attr_accessor :ver, :exceptions, :severity_level, :problem_id, :properties, + :measurements + + attribute_mapping( + ver: 'ver', + exceptions: 'exceptions', + severity_level: 'severityLevel', + problem_id: 'problemId', + properties: 'properties', + measurements: 'measurements' + ) + + def ver + @ver ||= 2 + end + + def exceptions + @exceptions ||= [] + end + + def properties + @properties ||= {} + end + + def measurements + @measurements ||= {} + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/exception_details.rb b/source/code/plugin/lib/application_insights/channel/contracts/exception_details.rb new file mode 100644 index 000000000..85bfc6282 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/exception_details.rb @@ -0,0 +1,28 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class ExceptionDetails + include JsonSerializable + + attr_accessor :id, :outer_id, :type_name, :message, :has_full_stack, :stack, + :parsed_stack + + attribute_mapping( + id: 'id', + outer_id: 'outerId', + type_name: 'typeName', + message: 'message', + has_full_stack: 'hasFullStack', + stack: 'stack', + parsed_stack: 'parsedStack' + ) + + def has_full_stack + @has_full_stack.nil? ? true : @has_full_stack + end + + def parsed_stack + @parsed_stack ||= [] + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/internal.rb b/source/code/plugin/lib/application_insights/channel/contracts/internal.rb new file mode 100644 index 000000000..6e8f3d300 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/internal.rb @@ -0,0 +1,15 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class Internal + include JsonSerializable + + attr_accessor :sdk_version, :agent_version, :node_name + + attribute_mapping( + sdk_version: 'ai.internal.sdkVersion', + agent_version: 'ai.internal.agentVersion', + node_name: 'ai.internal.nodeName' + ) + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/json_serializable.rb b/source/code/plugin/lib/application_insights/channel/contracts/json_serializable.rb new file mode 100644 index 000000000..8f4677044 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/json_serializable.rb @@ -0,0 +1,59 @@ +require 'json' + +module ApplicationInsights + module Channel + module Contracts + module JsonSerializable + module ClassMethods + attr_reader :json_mappings + + def attribute_mapping(mappings = {}) + @json_mappings = mappings + end + end + + def self.included(klass) + klass.extend JsonSerializable::ClassMethods + end + + def initialize(attributes = {}) + attributes.each { |k, v| send(:"#{k}=", v) } + end + + def to_h + output = {} + klass = self.class + + klass.json_mappings.each do |attr, name| + value = visit self.send(attr) + is_empty = value.respond_to?(:empty?) && value.empty? + + output[name] = value unless value.nil? || is_empty + end + + output + end + + def to_json(args = {}) + JSON.generate self.to_h, args + end + + private + + def visit(object) + return if object.nil? + + if object.is_a? Array + object.map { |e| visit e } + elsif object.is_a? Hash + Hash[object.map { |k, v| [k, visit(v)] }] + elsif object.respond_to? :to_h + object.to_h + else + object + end + end + end + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/location.rb b/source/code/plugin/lib/application_insights/channel/contracts/location.rb new file mode 100644 index 000000000..4136c869b --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/location.rb @@ -0,0 +1,13 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class Location + include JsonSerializable + + attr_accessor :ip + + attribute_mapping( + ip: 'ai.location.ip' + ) + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/message_data.rb b/source/code/plugin/lib/application_insights/channel/contracts/message_data.rb new file mode 100644 index 000000000..1340f5ba7 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/message_data.rb @@ -0,0 +1,24 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class MessageData + include JsonSerializable + + attr_accessor :ver, :message, :severity_level, :properties + + attribute_mapping( + ver: 'ver', + message: 'message', + severity_level: 'severityLevel', + properties: 'properties' + ) + + def ver + @ver ||= 2 + end + + def properties + @properties ||= {} + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/metric_data.rb b/source/code/plugin/lib/application_insights/channel/contracts/metric_data.rb new file mode 100644 index 000000000..bcb5739d6 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/metric_data.rb @@ -0,0 +1,27 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class MetricData + include JsonSerializable + + attr_accessor :ver, :metrics, :properties + + attribute_mapping( + ver: 'ver', + metrics: 'metrics', + properties: 'properties' + ) + + def ver + @ver ||= 2 + end + + def metrics + @metrics ||= [] + end + + def properties + @properties ||= {} + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/operation.rb b/source/code/plugin/lib/application_insights/channel/contracts/operation.rb new file mode 100644 index 000000000..c86dd111b --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/operation.rb @@ -0,0 +1,17 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class Operation + include JsonSerializable + + attr_accessor :id, :name, :parent_id, :synthetic_source, :correlation_vector + + attribute_mapping( + id: 'ai.operation.id', + name: 'ai.operation.name', + parent_id: 'ai.operation.parentId', + synthetic_source: 'ai.operation.syntheticSource', + correlation_vector: 'ai.operation.correlationVector' + ) + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/page_view_data.rb b/source/code/plugin/lib/application_insights/channel/contracts/page_view_data.rb new file mode 100644 index 000000000..d17dd2f79 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/page_view_data.rb @@ -0,0 +1,33 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class PageViewData + include JsonSerializable + + attr_accessor :ver, :url, :name, :duration, :id, :referrer_uri, :properties, + :measurements + + attribute_mapping( + ver: 'ver', + url: 'url', + name: 'name', + duration: 'duration', + id: 'id', + referrer_uri: 'referrerUri', + properties: 'properties', + measurements: 'measurements' + ) + + def ver + @ver ||= 2 + end + + def properties + @properties ||= {} + end + + def measurements + @measurements ||= {} + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/page_view_perf_data.rb b/source/code/plugin/lib/application_insights/channel/contracts/page_view_perf_data.rb new file mode 100644 index 000000000..adde3f3ad --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/page_view_perf_data.rb @@ -0,0 +1,39 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class PageViewPerfData + include JsonSerializable + + attr_accessor :ver, :url, :perf_total, :name, :duration, :network_connect, + :sent_request, :received_response, :id, :dom_processing, :referrer_uri, + :properties, :measurements + + attribute_mapping( + ver: 'ver', + url: 'url', + perf_total: 'perfTotal', + name: 'name', + duration: 'duration', + network_connect: 'networkConnect', + sent_request: 'sentRequest', + received_response: 'receivedResponse', + id: 'id', + dom_processing: 'domProcessing', + referrer_uri: 'referrerUri', + properties: 'properties', + measurements: 'measurements' + ) + + def ver + @ver ||= 2 + end + + def properties + @properties ||= {} + end + + def measurements + @measurements ||= {} + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/remote_dependency_data.rb b/source/code/plugin/lib/application_insights/channel/contracts/remote_dependency_data.rb new file mode 100644 index 000000000..a238841f6 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/remote_dependency_data.rb @@ -0,0 +1,40 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class RemoteDependencyData + include JsonSerializable + + attr_accessor :ver, :name, :id, :result_code, :duration, :success, :data, + :target, :type, :properties, :measurements + + attribute_mapping( + ver: 'ver', + name: 'name', + id: 'id', + result_code: 'resultCode', + duration: 'duration', + success: 'success', + data: 'data', + target: 'target', + type: 'type', + properties: 'properties', + measurements: 'measurements' + ) + + def ver + @ver ||= 2 + end + + def success + @success.nil? ? true : @success + end + + def properties + @properties ||= {} + end + + def measurements + @measurements ||= {} + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/reopenings.rb b/source/code/plugin/lib/application_insights/channel/contracts/reopenings.rb new file mode 100644 index 000000000..394bf8afb --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/reopenings.rb @@ -0,0 +1,27 @@ +module ApplicationInsights::Channel::Contracts + class ExceptionData + def handled_at + @properties["handledAt"] if @properties + end + + def handled_at=(handled_at) + if handled_at + @properties ||= {} + @properties["handledAt"] = handled_at + end + end + end + + class RequestData + def http_method + @properties["httpMethod"] if @properties + end + + def http_method=(http_method) + if http_method + @properties ||= {} + @properties["httpMethod"] = http_method + end + end + end +end \ No newline at end of file diff --git a/source/code/plugin/lib/application_insights/channel/contracts/request_data.rb b/source/code/plugin/lib/application_insights/channel/contracts/request_data.rb new file mode 100644 index 000000000..af2581c2b --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/request_data.rb @@ -0,0 +1,35 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class RequestData + include JsonSerializable + + attr_accessor :ver, :id, :source, :name, :duration, :response_code, :success, + :url, :properties, :measurements + + attribute_mapping( + ver: 'ver', + id: 'id', + source: 'source', + name: 'name', + duration: 'duration', + response_code: 'responseCode', + success: 'success', + url: 'url', + properties: 'properties', + measurements: 'measurements' + ) + + def ver + @ver ||= 2 + end + + def properties + @properties ||= {} + end + + def measurements + @measurements ||= {} + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/session.rb b/source/code/plugin/lib/application_insights/channel/contracts/session.rb new file mode 100644 index 000000000..a761c51c5 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/session.rb @@ -0,0 +1,14 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class Session + include JsonSerializable + + attr_accessor :id, :is_first + + attribute_mapping( + id: 'ai.session.id', + is_first: 'ai.session.isFirst' + ) + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/severity_level.rb b/source/code/plugin/lib/application_insights/channel/contracts/severity_level.rb new file mode 100644 index 000000000..322a00ec3 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/severity_level.rb @@ -0,0 +1,13 @@ +module ApplicationInsights::Channel::Contracts + class SeverityLevel + VERBOSE = 0 + + INFORMATION = 1 + + WARNING = 2 + + ERROR = 3 + + CRITICAL = 4 + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/stack_frame.rb b/source/code/plugin/lib/application_insights/channel/contracts/stack_frame.rb new file mode 100644 index 000000000..b4f4b9844 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/stack_frame.rb @@ -0,0 +1,17 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class StackFrame + include JsonSerializable + + attr_accessor :level, :method, :assembly, :file_name, :line + + attribute_mapping( + level: 'level', + method: 'method', + assembly: 'assembly', + file_name: 'fileName', + line: 'line' + ) + end +end diff --git a/source/code/plugin/lib/application_insights/channel/contracts/user.rb b/source/code/plugin/lib/application_insights/channel/contracts/user.rb new file mode 100644 index 000000000..a7ff8a7cf --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/contracts/user.rb @@ -0,0 +1,15 @@ +require_relative 'json_serializable' + +module ApplicationInsights::Channel::Contracts + class User + include JsonSerializable + + attr_accessor :account_id, :id, :auth_user_id + + attribute_mapping( + account_id: 'ai.user.accountId', + id: 'ai.user.id', + auth_user_id: 'ai.user.authUserId' + ) + end +end diff --git a/source/code/plugin/lib/application_insights/channel/event.rb b/source/code/plugin/lib/application_insights/channel/event.rb new file mode 100644 index 000000000..ae61064f8 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/event.rb @@ -0,0 +1,68 @@ +require_relative 'queue_base' +require 'thread' + +module ApplicationInsights + module Channel + # An event class that allows simple cross-thread signalling. + # + # An object of this type managers an internal flag that can be set to true + # via the {#set} method and reset via the {#clear} method. Calling the + # {#wait} method will block until the flag is set to true. + # + # @example + # require 'application_insights' + # require 'thread' + # event = ApplicationInsights::Channel::Event.new + # Thread.new do + # sleep 1 + # event.set + # end + # puts 'Main screen turn on.' + # result = event.wait + # puts 'All your base are belong to us.' + class Event + # Initializes a new instance of the class. + def initialize + @mutex = Mutex.new + @condition_variable = ConditionVariable.new + @signal = false + end + + # The signal value for this object. Note that the value of this property is + # not synchronized with respect to {#set} and {#clear} meaning that it + # could return false positives or negatives. + # @return [Boolean] the signal value. + attr_reader :signal + + # Sets the internal flag to true. Calling this method will also cause all + # waiting threads to awaken. + def set + @mutex.synchronize do + @signal = true + @condition_variable.broadcast + end + end + + # Sets the internal flag to false. + def clear + @mutex.synchronize do + @signal = false + end + end + + # Calling this method will block until the internal flag is set to true. + # If the flag is set to true before calling this method, we will return + # immediately. If the timeout parameter is specified, the method will + # unblock after the specified number of seconds. + # @param [Fixnum] timeout the timeout for the operation in seconds. + # @return [Boolean] the value of the internal flag on exit. + def wait(timeout=nil) + @mutex.synchronize do + @condition_variable.wait(@mutex, timeout) unless @signal + end + + @signal + end + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/queue_base.rb b/source/code/plugin/lib/application_insights/channel/queue_base.rb new file mode 100644 index 000000000..91226b17f --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/queue_base.rb @@ -0,0 +1,73 @@ +require 'thread' + +module ApplicationInsights + module Channel + # The base class for all types of queues for use in conjunction with an + # implementation of {SenderBase}. The queue will notify the sender that it + # needs to pick up items when it reaches {#max_queue_length}, or when the + # consumer calls {#flush}. + class QueueBase + # Initializes a new instance of the class. + # @param [SenderBase] sender the sender object that will be used in + # conjunction with this queue. + def initialize(sender) + @queue = Queue.new + @max_queue_length = 500 + self.sender = sender + end + + # The maximum number of items that will be held by the queue before the + # queue will call the {#flush} method. + # @return [Fixnum] the maximum queue size. (defaults to: 500) + attr_accessor :max_queue_length + + # The sender that is associated with this queue that this queue will use to + # send data to the service. + # @return [SenderBase] the sender object. + attr_reader :sender + + # Change the sender that is associated with this queue. + # @param [SenderBase] sender the sender object. + # @return [SenderBase] the sender object. + def sender=(sender) + @sender = sender + @sender.queue = self if sender + @sender + end + + # Adds the passed in item object to the queue and calls {#flush} if the + # size of the queue is larger than {#max_queue_length}. This method does + # nothing if the passed in item is nil. + # @param [Contracts::Envelope] item the telemetry envelope object to send + # to the service. + def push(item) + return unless item + + @queue.push(item) + + flush if @queue.length >= @max_queue_length + end + + # Pops a single item from the queue and returns it. If the queue is empty, + # this method will return nil. + # @return [Contracts::Envelope] a telemetry envelope object or nil if the + # queue is empty. + def pop + return @queue.pop(true) + rescue ThreadError + return nil + end + + # Flushes the current queue by notifying the {#sender}. This method needs + # to be overridden by a concrete implementations of the queue class. + def flush + end + + # Indicates whether the queue is empty. + # @return [Boolean] true if the queue is empty + def empty? + @queue.empty? + end + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/sender_base.rb b/source/code/plugin/lib/application_insights/channel/sender_base.rb new file mode 100644 index 000000000..2431bf748 --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/sender_base.rb @@ -0,0 +1,88 @@ +require 'json' +require 'net/http' +require 'openssl' +require 'stringio' +require 'zlib' +require 'logger' + +module ApplicationInsights + module Channel + # The base class for all types of senders for use in conjunction with an + # implementation of {QueueBase}. The queue will notify the sender that it + # needs to pick up items. The concrete sender implementation will listen to + # these notifications and will pull items from the queue using + # {QueueBase#pop} getting at most {#send_buffer_size} items. + # It will then call {#send} using the list of items pulled from the queue. + class SenderBase + # Initializes a new instance of the class. + # @param [String] service_endpoint_uri the address of the service to send + # telemetry data to. + def initialize(service_endpoint_uri) + @service_endpoint_uri = service_endpoint_uri + @queue = nil + @send_buffer_size = 100 + @logger = Logger.new(STDOUT) + end + + # The service endpoint URI where this sender will send data to. + # @return [String] the service endpoint URI. + attr_accessor :service_endpoint_uri + + # The queue that this sender is draining. While {SenderBase} doesn't + # implement any means of doing so, derivations of this class do. + # @return [QueueBase] the queue instance that this sender is draining. + attr_accessor :queue + + # The buffer size for a single batch of telemetry. This is the maximum number + # of items in a single service request that this sender is going to send. + # @return [Fixnum] the maximum number of items in a telemetry batch. + attr_accessor :send_buffer_size + + # The logger for the sender. + attr_accessor :logger + + # Immediately sends the data passed in to {#service_endpoint_uri}. If the + # service request fails, the passed in items are pushed back to the {#queue}. + # @param [Array] data_to_send an array of + # {Contracts::Envelope} objects to send to the service. + def send(data_to_send) + uri = URI(@service_endpoint_uri) + headers = { + 'Accept' => 'application/json', + 'Content-Type' => 'application/json; charset=utf-8', + 'Content-Encoding' => 'gzip' + } + request = Net::HTTP::Post.new(uri.path, headers) + + # Use JSON.generate instead of to_json, otherwise it will + # default to ActiveSupport::JSON.encode for Rails app + json = JSON.generate(data_to_send) + compressed_data = compress(json) + request.body = compressed_data + + http = Net::HTTP.new uri.hostname, uri.port + if uri.scheme.downcase == 'https' + http.use_ssl = true + http.verify_mode = OpenSSL::SSL::VERIFY_NONE + end + + response = http.request(request) + http.finish if http.started? + + if !response.kind_of? Net::HTTPSuccess + @logger.warn('application_insights') { "Failed to send data: #{response.message}" } + end + end + + private + + def compress(string) + wio = StringIO.new("w") + w_gz = Zlib::GzipWriter.new wio, nil, nil + w_gz.write(string) + w_gz.close + wio.string + end + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/synchronous_queue.rb b/source/code/plugin/lib/application_insights/channel/synchronous_queue.rb new file mode 100644 index 000000000..13c2281ac --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/synchronous_queue.rb @@ -0,0 +1,45 @@ +require_relative 'queue_base' + +module ApplicationInsights + module Channel + # A synchronous queue for use in conjunction with the {SynchronousSender}. + # The queue will call {SenderBase#send} when it reaches {#max_queue_length}, + # or when the consumer calls {#flush}. + # + # @example + # require 'application_insights' + # require 'thread' + # queue = ApplicationInsights::Channel::SynchronousQueue.new nil + # queue.max_queue_length = 1 + # queue.push 1 + class SynchronousQueue < QueueBase + # Initializes a new instance of the class. + # @param [SenderBase] sender the sender object that will be used in + # conjunction with this queue. + def initialize(sender) + super sender + end + + # Flushes the current queue by by calling {#sender}'s + # {SenderBase#send} method. + def flush + local_sender = @sender + return unless local_sender + + while true + # get at most send_buffer_size items and send them + data = [] + while data.length < local_sender.send_buffer_size + item = pop() + break if not item + data.push item + end + + break if data.length == 0 + + local_sender.send(data) + end + end + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/synchronous_sender.rb b/source/code/plugin/lib/application_insights/channel/synchronous_sender.rb new file mode 100644 index 000000000..ade2f086c --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/synchronous_sender.rb @@ -0,0 +1,17 @@ +require_relative 'sender_base' + +module ApplicationInsights + module Channel + # A synchronous sender that works in conjunction with the {SynchronousQueue}. + # The queue will call {#send} on the current instance with the data to send. + class SynchronousSender < SenderBase + SERVICE_ENDPOINT_URI = 'https://dc.services.visualstudio.com/v2/track' + # Initializes a new instance of the class. + # @param [String] service_endpoint_uri the address of the service to send + # telemetry data to. + def initialize(service_endpoint_uri = SERVICE_ENDPOINT_URI) + super service_endpoint_uri + end + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/telemetry_channel.rb b/source/code/plugin/lib/application_insights/channel/telemetry_channel.rb new file mode 100644 index 000000000..e026ebf7d --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/telemetry_channel.rb @@ -0,0 +1,131 @@ +require 'time' +require_relative 'asynchronous_queue' +require_relative 'asynchronous_sender' +require_relative 'telemetry_context' +require_relative 'synchronous_queue' +require_relative 'synchronous_sender' +require_relative 'contracts/envelope' +require_relative 'contracts/data' +require_relative 'contracts/internal' +require_relative '../../application_insights/version' + +module ApplicationInsights + module Channel + # The telemetry channel is responsible for constructing a + # {Contracts::Envelope} object from the passed in data and specified + # telemetry context. + # + # @example + # require 'application_insights' + # channel = ApplicationInsights::Channel::TelemetryChannel.new + # event = ApplicationInsights::Channel::Contracts::EventData.new name: 'My event' + # channel.write event + class TelemetryChannel + # Initializes a new instance of the class. + # @param [TelemetryContext] context the telemetry context to use when + # sending telemetry data. + # @param [QueueBase] queue the queue to enqueue the resulting + # {Contracts::Envelope} to. + def initialize(context=nil, queue=nil) + @context = context || TelemetryContext.new + @queue = queue || SynchronousQueue.new(SynchronousSender.new) + end + + # The context associated with this channel. All {Contracts::Envelope} + # objects created by this channel will use this value if it's present or if + # none is specified as part of the {#write} call. + # @return [TelemetryContext] the context instance + # (defaults to: TelemetryContext.new) + attr_reader :context + + # The queue associated with this channel. All {Contracts::Envelope} objects + # created by this channel will be pushed to this queue. + # @return [QueueBase] the queue instance (defaults to: SynchronousQueue.new) + attr_reader :queue + + # The sender associated with this channel. This instance will be used to + # transmit telemetry to the service. + # @return [SenderBase] the sender instance (defaults to: SynchronousSender.new) + def sender + @queue.sender + end + + # Flushes the enqueued data by calling {QueueBase#flush}. + def flush + @queue.flush + end + + # Enqueues the passed in data to the {#queue}. If the caller specifies a + # context as well, it will take precedence over the instance in {#context}. + # @param [Object] data the telemetry data to send. This will be wrapped in + # an {Contracts::Envelope} before being enqueued to the {#queue}. + # @param [TelemetryContext] context the override context to use when + # constructing the {Contracts::Envelope}. + # @param [Time|String] time the timestamp of the telemetry used to construct the + # {Contracts::Envelope}. + def write(data, context=nil, time=nil) + local_context = context || @context + raise ArgumentError, 'Context was required but not provided' unless local_context + + if time && time.is_a?(String) + local_time = time + elsif time && time.is_a?(Time) + local_time = time.iso8601(7) + else + local_time = Time.now.iso8601(7) + end + + data_type = data.class.name.gsub(/^.*::/, '') + set_properties data, local_context + data_attributes = { + :base_type => data_type, + :base_data => data + } + envelope_attributes = { + :name => 'Microsoft.ApplicationInsights.' + data_type[0..-5], + :time => local_time, + :i_key => local_context.instrumentation_key, + :tags => get_tags(local_context), + :data => Contracts::Data.new(data_attributes) + } + envelope = Contracts::Envelope.new envelope_attributes + @queue.push(envelope) + end + + private + + def get_tags(context) + hash = {} + internal_context_attributes = { + :sdk_version => 'rb:' + ApplicationInsights::VERSION + } + internal_context = Contracts::Internal.new internal_context_attributes + + [internal_context, + context.application, + context.cloud, + context.device, + context.user, + context.session, + context.location, + context.operation].each { |c| hash.merge!(c.to_h) if c } + + hash.delete_if { |k, v| v.nil? } + + hash + end + + def set_properties(data, context) + if context.properties + properties = data.properties || {} + context.properties.each do |key, value| + unless properties.key?(key) + properties[key] = value + end + end + data.properties = properties + end + end + end + end +end diff --git a/source/code/plugin/lib/application_insights/channel/telemetry_context.rb b/source/code/plugin/lib/application_insights/channel/telemetry_context.rb new file mode 100644 index 000000000..bb24af24e --- /dev/null +++ b/source/code/plugin/lib/application_insights/channel/telemetry_context.rb @@ -0,0 +1,85 @@ +require_relative 'contracts/application' +require_relative 'contracts/cloud' +require_relative 'contracts/device' +require_relative 'contracts/user' +require_relative 'contracts/session' +require_relative 'contracts/operation' +require_relative 'contracts/location' + +module ApplicationInsights + module Channel + # Represents the context for sending telemetry to the + # Application Insights service. + # + # @example + # require 'application_insights' + # context = ApplicationInsights::Channel::TelemetryContext.new + # context.instrumentation_key = '' + # context.application.id = 'My application' + # context.application.ver = '1.2.3' + # context.device.id = 'My current device' + # context.device.oem_name = 'Asus' + # context.device.model = 'X31A' + # context.device.type = "Other" + # context.user.id = 'santa@northpole.net' + class TelemetryContext + # Initializes a new instance of the class. + def initialize + @instrumentation_key = nil + @application = Contracts::Application.new + @cloud = Contracts::Cloud.new + @device = Contracts::Device.new + @user = Contracts::User.new + @session = Contracts::Session.new + @operation = Contracts::Operation.new + @location = Contracts::Location.new + @properties = {} + end + + # The instrumentation key that is used to identify which + # Application Insights application this data is for. + # @return [String] the instrumentation key. + attr_accessor :instrumentation_key + + # The application context. This contains properties of the + # application you are running. + # @return [Contracts::Application] the context object. + attr_accessor :application + + # The cloud context. This contains properties of the + # cloud role you are generating telemetry for. + # @return [Contracts::Cloud] the context object. + attr_accessor :cloud + + # The device context. This contains properties of the + # device you are running on. + # @return [Contracts::Device] the context object. + attr_accessor :device + + # The user context. This contains properties of the + # user you are generating telemetry for. + # @return [Contracts::User] the context object. + attr_accessor :user + + # The session context. This contains properties of the + # session you are generating telemetry for. + # @return [Contracts::Session] the context object. + attr_accessor :session + + # The operation context. This contains properties of the + # operation you are generating telemetry for. + # @return [Contracts::Operation] the context object. + attr_accessor :operation + + # The location context. This contains properties of the + # location you are generating telemetry from. + # @return [Contracts::Location] the context object. + attr_accessor :location + + # The property context. This contains free-form properties + # that you can add to your telemetry. + # @return [Hash] the context object. + attr_accessor :properties + end + end +end diff --git a/source/code/plugin/lib/application_insights/rack/track_request.rb b/source/code/plugin/lib/application_insights/rack/track_request.rb new file mode 100644 index 000000000..62c2b0844 --- /dev/null +++ b/source/code/plugin/lib/application_insights/rack/track_request.rb @@ -0,0 +1,154 @@ +require 'rack' +require 'securerandom' +require_relative '../channel/contracts/request_data' +require_relative '../telemetry_client' + +module ApplicationInsights + module Rack + # Track every request and sends the request data to Application Insights. + class TrackRequest + # Initializes a new instance of the class. + # @param [Object] app the inner rack application. + # @param [String] instrumentation_key to identify which Application Insights + # application this data is for. + # @param [Fixnum] buffer_size the buffer size and the buffered requests would + # send to Application Insights when buffer is full. + # @param [Fixnum] send_interval the frequency (in seconds) to check buffer + # and send buffered requests to Application Insights if any. + def initialize(app, instrumentation_key, buffer_size = 500, send_interval = 60) + @app = app + @instrumentation_key = instrumentation_key + @buffer_size = buffer_size + @send_interval = send_interval + + @sender = Channel::AsynchronousSender.new + @sender.send_interval = @send_interval + queue = Channel::AsynchronousQueue.new @sender + queue.max_queue_length = @buffer_size + @channel = Channel::TelemetryChannel.new nil, queue + + @client = TelemetryClient.new @instrumentation_key, @channel + end + + # Track requests and send data to Application Insights asynchronously. + # @param [Hash] env the rack environment. + def call(env) + # Build a request ID, incorporating one from our request if one exists. + request_id = request_id_header(env['HTTP_REQUEST_ID']) + env['ApplicationInsights.request.id'] = request_id + + start = Time.now + begin + status, headers, response = @app.call(env) + rescue Exception => ex + status = 500 + exception = ex + end + stop = Time.now + + start_time = start.iso8601(7) + duration = format_request_duration(stop - start) + success = status.to_i < 400 + + request = ::Rack::Request.new env + options = options_hash(request) + + data = request_data(request_id, start_time, duration, status, success, options) + context = telemetry_context(request_id, env['HTTP_REQUEST_ID']) + + @client.channel.write data, context, start_time + + if exception + @client.track_exception exception, handled_at: 'Unhandled' + raise exception + end + + [status, headers, response] + end + + private + + def sender=(sender) + if sender.is_a? Channel::AsynchronousSender + @sender = sender + @client.channel.queue.sender = @sender + end + end + + def client + @client + end + + def format_request_duration(duration_seconds) + if duration_seconds >= 86400 + # just return 1 day when it takes more than 1 day which should not happen for requests. + return "%02d.%02d:%02d:%02d.%07d" % [1, 0, 0, 0, 0] + end + + Time.at(duration_seconds).gmtime.strftime("00.%H:%M:%S.%7N") + end + + def request_id_header(request_id) + valid_request_id_header = valid_request_id(request_id) + + length = valid_request_id_header ? 5 : 10 + id = SecureRandom.base64(length) + + if valid_request_id_header + request_id_has_end = %w[. _].include?(request_id[-1]) + request_id << '.' unless request_id_has_end + + return "#{request_id}#{id}_" + end + + "|#{id}." + end + + def valid_request_id(request_id) + request_id && request_id[0] == '|' + end + + def operation_id(id) + # Returns the root ID from the '|' to the first '.' if any. + root_start = id[0] == '|' ? 1 : 0 + + root_end = id.index('.') + root_end = root_end ? root_end - 1 : id.length - root_start + + id[root_start..root_end] + end + + def options_hash(request) + { + name: "#{request.request_method} #{request.path}", + http_method: request.request_method, + url: request.url + } + end + + def request_data(request_id, start_time, duration, status, success, options) + Channel::Contracts::RequestData.new( + :id => request_id || 'Null', + :duration => duration || '0:00:00:00.0000000', + :response_code => status || 200, + :success => success == nil ? true : success, + :name => options[:name], + :url => options[:url], + :properties => options[:properties] || {}, + :measurements => options[:measurements] || {}, + # Must initialize http_method after properties because it's actually stored in properties + :http_method => options[:http_method] + ) + end + + def telemetry_context(request_id, request_id_header) + context = Channel::TelemetryContext.new + context.instrumentation_key = @instrumentation_key + context.operation.id = operation_id(request_id) + context.operation.parent_id = request_id_header + + context + end + end + end +end diff --git a/source/code/plugin/lib/application_insights/telemetry_client.rb b/source/code/plugin/lib/application_insights/telemetry_client.rb new file mode 100644 index 000000000..bd066ae70 --- /dev/null +++ b/source/code/plugin/lib/application_insights/telemetry_client.rb @@ -0,0 +1,232 @@ +require_relative 'channel/telemetry_context' +require_relative 'channel/telemetry_channel' +require_relative 'channel/contracts/page_view_data' +require_relative 'channel/contracts/remote_dependency_data' +require_relative 'channel/contracts/exception_data' +require_relative 'channel/contracts/exception_details' +require_relative 'channel/contracts/event_data' +require_relative 'channel/contracts/data_point' +require_relative 'channel/contracts/data_point_type' +require_relative 'channel/contracts/metric_data' +require_relative 'channel/contracts/message_data' +require_relative 'channel/contracts/stack_frame' +require_relative 'channel/contracts/request_data' +require_relative 'channel/contracts/severity_level' +require_relative 'channel/contracts/reopenings' + +module ApplicationInsights + # The telemetry client used for sending all types of telemetry. It serves as + # the main entry point for interacting with the Application Insights service. + class TelemetryClient + # Initializes a new instance of the class. + # @param [String] instrumentation_key to identify which Application Insights + # application this data is for. + # @param [Channel::TelemetryChannel] telemetry_channel the optional telemetry + # channel to be used instead of constructing a default one. + def initialize(instrumentation_key = nil, telemetry_channel = nil) + @context = Channel::TelemetryContext.new + @context.instrumentation_key = instrumentation_key + @channel = telemetry_channel || Channel::TelemetryChannel.new + end + + # The context associated with this client. All data objects created by this + # client will be accompanied by this value. + # @return [Channel::TelemetryContext] the context instance. + attr_reader :context + + # The channel associated with this telemetry client. All data created by this + # client will be passed along with the {#context} object to + # {Channel::TelemetryChannel#write} + # @return [Channel::TelemetryChannel] the channel instance. + attr_reader :channel + + # Send information about the page viewed in the application (a web page for + # instance). + # @param [String] name the name of the page that was viewed. + # @param [String] url the URL of the page that was viewed. + # @param [Hash] options the options to create the + # {Channel::Contracts::PageViewData} object. + # @option options [Fixnum] :duration the duration of the page view in + # milliseconds. (defaults to: 0) + # @option options [Hash] :properties the set of custom properties the client + # wants attached to this data item. (defaults to: {}) + # @option options [Hash] :measurements the set of custom measurements the + # client wants to attach to this data item (defaults to: {}) + def track_page_view(name, url, options={}) + data_attributes = { + :name => name || 'Null', + :url => url, + :duration => options[:duration], + :properties => options[:properties] || {}, + :measurements => options[:measurements] || {} + } + data = Channel::Contracts::PageViewData.new data_attributes + self.channel.write(data, self.context) + end + + # Send information about a single exception that occurred in the application. + # @param [Exception] exception the exception that the client wants to send. + # @param [Hash] options the options to create the + # {Channel::Contracts::ExceptionData} object. + # @option options [String] :handled_at the type of exception + # (defaults to: 'UserCode') + # @option options [Hash] :properties the set of custom properties the client + # wants attached to this data item. (defaults to: {}) + # @option options [Hash] :measurements the set of custom measurements the + # client wants to attach to this data item (defaults to: {}) + def track_exception(exception, options={}) + return unless exception.is_a? Exception + + parsed_stack = [] + if exception.backtrace + frame_pattern = /^(?.*):(?\d+)(\.|:in `((?.*)'$))/ + + exception.backtrace.each_with_index do |frame, counter| + match = frame_pattern.match frame + stack_frame = Channel::Contracts::StackFrame.new( + :assembly => 'Unknown', + :file_name => match['file'], + :level => counter, + :line => match['line'], + :method => match['method'] + ) + + parsed_stack << stack_frame + end + end + + details = Channel::Contracts::ExceptionDetails.new( + :id => 1, + :outer_id => 0, + :type_name => exception.class.name, + :message => exception.message, + :has_full_stack => exception.backtrace != nil, + :stack => (exception.backtrace.join("\n") if exception.backtrace), + :parsed_stack => parsed_stack + ) + + data = Channel::Contracts::ExceptionData.new( + :exceptions => [details], + :properties => options[:properties] || {}, + :measurements => options[:measurements] || {}, + # Must initialize handled_at after properties because it's actually stored in properties + :handled_at => options.fetch(:handled_at, 'UserCode') + ) + + self.channel.write(data, self.context) + end + + # Send information about a single event that has occurred in the context of + # the application. + # @param [String] name the data to associate to this event. + # @param [Hash] options the options to create the + # {Channel::Contracts::EventData} object. + # @option options [Hash] :properties the set of custom properties the client + # wants attached to this data item. (defaults to: {}) + # @option options [Hash] :measurements the set of custom measurements the + # client wants to attach to this data item (defaults to: {}) + def track_event(name, options={}) + data = Channel::Contracts::EventData.new( + :name => name || 'Null', + :properties => options[:properties] || {}, + :measurements => options[:measurements] || {} + ) + + self.channel.write(data, self.context) + end + + # Send information about a single metric data point that was captured for + # the application. + # @param [String] name the name of the metric that was captured. + # @param [Fixnum] value the value of the metric that was captured. + # @param [Hash] options the options to create the + # {Channel::Contracts::MetricData} object. + # @option options [Channel::Contracts::DataPointType] :type the type of the + # metric (defaults to: {Channel::Contracts::DataPointType::AGGREGATION}) + # @option options [Fixnum] :count the number of metrics that were aggregated + # into this data point (defaults to: 0) + # @option options [Fixnum] :min the minimum of all metrics collected that + # were aggregated into this data point (defaults to: 0) + # @option options [Fixnum] :max the maximum of all metrics collected that + # were aggregated into this data point (defaults to: 0) + # @option options [Fixnum] :std_dev the standard deviation of all metrics + # collected that were aggregated into this data point (defaults to: 0) + # @option options [Hash] :properties the set of custom properties the client + # wants attached to this data item. (defaults to: {}) + # @option options [Hash] :measurements the set of custom measurements the + # client wants to attach to this data item (defaults to: {}) + def track_metric(name, value, options={}) + data_point = Channel::Contracts::DataPoint.new( + :name => name || 'Null', + :value => value || 0, + :kind => options[:type] || Channel::Contracts::DataPointType::AGGREGATION, + :count => options[:count], + :min => options[:min], + :max => options[:max], + :std_dev => options[:std_dev] + ) + + data = Channel::Contracts::MetricData.new( + :metrics => [data_point], + :properties => options[:properties] || {} + ) + + self.channel.write(data, self.context) + end + + # Sends a single trace statement. + # @param [String] name the trace statement. + # @param [Channel::Contracts::SeverityLevel] severity_level the severity level. + # @param [Hash] options the options to create the + # {Channel::Contracts::EventData} object. + # @option options [Hash] :properties the set of custom properties the client + # wants attached to this data item. (defaults to: {}) + def track_trace(name, severity_level = nil, options={}) + data = Channel::Contracts::MessageData.new( + :message => name || 'Null', + :severity_level => severity_level || Channel::Contracts::SeverityLevel::INFORMATION, + :properties => options[:properties] || {} + ) + + self.channel.write(data, self.context) + end + + # Sends a single request. + # @param [String] id the unique identifier of the request. + # @param (String) start_time the start time of the request. + # @param [String] duration the duration to process the request. + # @param [String] response_code the response code of the request. + # @param [Boolean] success indicates whether the request succeeds or not. + # @param [Hash] options the options to create the + # {Channel::Contracts::RequestData} object. + # @option options [String] :name the name of the request. + # @option options [String] :http_method the http method used for the request. + # @option options [String] :url the url of the request. + # @option options [Hash] :properties the set of custom properties the client + # wants attached to this data item. (defaults to: {}) + # @option options [Hash] :measurements the set of custom measurements the + # client wants to attach to this data item (defaults to: {}) + def track_request(id, start_time, duration, response_code, success, options={}) + data = Channel::Contracts::RequestData.new( + :id => id || 'Null', + :duration => duration || '0:00:00:00.0000000', + :response_code => response_code || 200, + :success => success = nil ? true : success, + :name => options[:name], + :url => options[:url], + :properties => options[:properties] || {}, + :measurements => options[:measurements] || {}, + # Must initialize http_method after properties because it's actually stored in properties + :http_method => options[:http_method] + ) + + self.channel.write(data, self.context, start_time) + end + + # Flushes data in the queue. Data in the queue will be sent either immediately + # irrespective of what sender is being used. + def flush + self.channel.flush + end + end +end diff --git a/source/code/plugin/lib/application_insights/unhandled_exception.rb b/source/code/plugin/lib/application_insights/unhandled_exception.rb new file mode 100644 index 000000000..aa87b6f85 --- /dev/null +++ b/source/code/plugin/lib/application_insights/unhandled_exception.rb @@ -0,0 +1,49 @@ +require_relative 'telemetry_client' +require_relative 'channel/telemetry_channel' +require_relative 'channel/synchronous_queue' +require_relative 'channel/synchronous_sender' + +include ApplicationInsights + +module ApplicationInsights + module UnhandledException + @sender = nil + + # Auto collects unhandled exception and send to the Application Insights service. + # @param (string) instrumentation_key used to identify which Application + # Insights application this data is for. + # @example + # require 'application_insights' + # ApplicationInsights::UnhandledException.collect('') + # raise Exception, 'Boom!' + def self.collect(instrumentation_key) + at_exit do + # Avoid sending exception more than once if this method got invoked multiple times + send(instrumentation_key) unless @sender + end + end + + # @api private + # Send the last raised exception to the Application Insights service if + # telemetry_sender is not customized. + # @param (string) instrumentation_key used to identify which Application + # Insights application this data is for. + # @param (SenderBase) telemetry_sender used to send the last raised exception. + def self.send(instrumentation_key, telemetry_sender = nil) + if $! && !$!.is_a?(SystemExit) && !$!.is_a?(SignalException) + if telemetry_sender + @sender = telemetry_sender + elsif !@sender + # Use a synchronized sender to guarantee the data would be sent out once flush + @sender = Channel::SynchronousSender.new + end + + queue = Channel::SynchronousQueue.new @sender + channel = Channel::TelemetryChannel.new nil, queue + client = TelemetryClient.new instrumentation_key, channel + client.track_exception($!, handled_at: 'Unhandled') + client.flush + end + end + end +end diff --git a/source/code/plugin/lib/application_insights/version.rb b/source/code/plugin/lib/application_insights/version.rb new file mode 100644 index 000000000..d2d56e833 --- /dev/null +++ b/source/code/plugin/lib/application_insights/version.rb @@ -0,0 +1,3 @@ +module ApplicationInsights + VERSION = '0.5.7'.freeze +end From 6c5d6cfd810bcc808f39b4f9094f6f6ad309af98 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 30 Oct 2018 17:56:19 -0700 Subject: [PATCH 31/45] changes --- .../code/plugin/ApplicationInsightsUtility.rb | 28 ++++++++++--------- source/code/plugin/DockerApiClient.rb | 6 ++-- source/code/plugin/in_containerinventory.rb | 1 - 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/source/code/plugin/ApplicationInsightsUtility.rb b/source/code/plugin/ApplicationInsightsUtility.rb index e25b7fa23..537b67c20 100644 --- a/source/code/plugin/ApplicationInsightsUtility.rb +++ b/source/code/plugin/ApplicationInsightsUtility.rb @@ -6,7 +6,7 @@ class ApplicationInsightsUtility require_relative 'omslog' require_relative 'DockerApiClient' require 'json' - require "base64" + require 'base64' @@HeartBeat = 'HeartBeatEvent' @@Exception = 'ExceptionEvent' @@ -16,11 +16,10 @@ class ApplicationInsightsUtility @OmsAdminFilePath = '/etc/opt/microsoft/omsagent/conf/omsadmin.conf' @@EnvAcsResourceName = 'ACS_RESOURCE_NAME' @@EnvAksRegion = 'AKS_REGION' - @@EnvAgentVersion = 'AgentVersion' + @@EnvAgentVersion = 'AGENTVERSION' + @@EnvApplicationInsightsKey = 'APPLICATIONINSIGHTS_AUTH' @@CustomProperties = {} - encodedAppInsightsKey = ENV['APPLICATIONINSIGHTS_AUTH'] - decodedAppInsightsKey = Base64.decode64(encodedAppInsightsKey) - @@Tc = ApplicationInsights::TelemetryClient.new decodedAppInsightsKey + @@Tc = nil def initialize end @@ -61,6 +60,9 @@ def initilizeutility() @@CustomProperties['DockerApiVersion'] = dockerInfo['ApiVersion'] @@CustomProperties['WorkspaceID'] = getWorkspaceId @@CustomProperties['AgentVersion'] = ENV[@@EnvAgentVersion] + encodedAppInsightsKey = ENV[@@encodedAppInsightsKey] + decodedAppInsightsKey = Base64.decode64(encodedAppInsightsKey) + @@Tc = ApplicationInsights::TelemetryClient.new decodedAppInsightsKey rescue => errorStr $log.warn("Exception in AppInsightsUtility: initilizeUtility - error: #{errorStr}") end @@ -69,8 +71,8 @@ def initilizeutility() def sendHeartBeatEvent(pluginName, properties) begin eventName = pluginName + @@HeartBeat - @@tc.track_event eventName , :properties => @@CustomProperties - @@tc.flush + @@Tc.track_event eventName , :properties => @@CustomProperties + @@Tc.flush rescue =>errorStr $log.warn("Exception in AppInsightsUtility: sendHeartBeatEvent - error: #{errorStr}") end @@ -78,23 +80,23 @@ def sendHeartBeatEvent(pluginName, properties) def sendCustomEvent(pluginName, properties) begin - @@tc.track_metric 'LastProcessedContainerInventoryCount', properties['ContainerCount'], + @@Tc.track_metric 'LastProcessedContainerInventoryCount', properties['ContainerCount'], :kind => ApplicationInsights::Channel::Contracts::DataPointType::MEASUREMENT, :properties => @@CustomProperties - @@tc.flush + @@Tc.flush rescue => errorStr $log.warn("Exception in AppInsightsUtility: sendCustomEvent - error: #{errorStr}") end end - def sendExceptionTelemetry(pluginName, errorStr) + def sendExceptionTelemetry(errorStr) begin if @@CustomProperties.empty? || @@CustomProperties.nil? initilizeutility end eventName = pluginName + @@Exception - @@tc.track_exception errorStr , :properties => @@CustomProperties - @@tc.flush + @@Tc.track_exception errorStr , :properties => @@CustomProperties + @@Tc.flush rescue => errorStr $log.warn("Exception in AppInsightsUtility: sendExceptionTelemetry - error: #{errorStr}") end @@ -108,7 +110,7 @@ def sendTelemetry(pluginName, properties) @@CustomProperties['Computer'] = properties['Computer'] sendHeartBeatEvent(pluginName, properties) sendCustomEvent(pluginName, properties) - rescue + rescue => errorStr $log.warn("Exception in AppInsightsUtility: sendTelemetry - error: #{errorStr}") end end diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index 2f03f30c8..5a73441da 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -13,7 +13,7 @@ class DockerApiClient @@SocketPath = "/var/run/docker.sock" @@ChunkSize = 4096 @@TimeoutInSeconds = 5 - + @@PluginName = 'ContainerInventory' def initialize end @@ -43,7 +43,7 @@ def getResponse(request, isMultiJson) return (isTimeOut)? nil : parseResponse(dockerResponse, isMultiJson) rescue => errorStr $log.warn("Socket call failed for request: #{request} error: #{errorStr} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") - ApplicationInsightsUtility.sendExceptionTelemetry('ContainerInventory', errorStr) + ApplicationInsightsUtility.sendExceptionTelemetry(@@PluginName + '-' + errorStr) end end @@ -62,7 +62,7 @@ def parseResponse(dockerResponse, isMultiJson) end rescue => errorStr $log.warn("Json parsing for docker response failed: #{errorStr} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") - ApplicationInsightsUtility.sendExceptionTelemetry('ContainerInventory', errorStr) + ApplicationInsightsUtility.sendExceptionTelemetry(@@PluginName + '-' + errorStr) end return parsedJsonResponse end diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 7072d3c2a..48c870430 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -18,7 +18,6 @@ def initialize require_relative 'DockerApiClient' require_relative 'ContainerInventoryState' require_relative 'omslog' - require_relative 'application_insights' end config_param :run_interval, :time, :default => '1m' From 9e192baf3121ecce6050395ad636c06f5427f442 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 30 Oct 2018 20:49:45 -0700 Subject: [PATCH 32/45] changes --- .../code/plugin/ApplicationInsightsUtility.rb | 42 +++++++++++-------- source/code/plugin/DockerApiClient.rb | 4 +- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/source/code/plugin/ApplicationInsightsUtility.rb b/source/code/plugin/ApplicationInsightsUtility.rb index 537b67c20..6bd770949 100644 --- a/source/code/plugin/ApplicationInsightsUtility.rb +++ b/source/code/plugin/ApplicationInsightsUtility.rb @@ -16,16 +16,16 @@ class ApplicationInsightsUtility @OmsAdminFilePath = '/etc/opt/microsoft/omsagent/conf/omsadmin.conf' @@EnvAcsResourceName = 'ACS_RESOURCE_NAME' @@EnvAksRegion = 'AKS_REGION' - @@EnvAgentVersion = 'AGENTVERSION' + @@EnvAgentVersion = 'AGENT_VERSION' @@EnvApplicationInsightsKey = 'APPLICATIONINSIGHTS_AUTH' @@CustomProperties = {} - @@Tc = nil def initialize end class << self - def initilizeutility() + #Set default properties for telemetry event + def initializeUtility() begin resourceInfo = ENV['AKS_RESOURCE_ID'] if resourceInfo.nil? || resourceInfo.empty? @@ -35,11 +35,9 @@ def initilizeutility() @@CustomProperties["ResourceGroupName"] = "" @@CustomProperties["ClusterName"] = "" @@CustomProperties["Region"] = "" - @@CustomProperties["AKS_RESOURCE_ID"] = "" else aksResourceId = ENV['AKS_RESOURCE_ID'] @@CustomProperties["AKS_RESOURCE_ID"] = aksResourceId - @@CustomProperties["ACSResourceName"] = "" begin splitStrings = aksResourceId.split('/') subscriptionId = splitStrings[2] @@ -60,9 +58,11 @@ def initilizeutility() @@CustomProperties['DockerApiVersion'] = dockerInfo['ApiVersion'] @@CustomProperties['WorkspaceID'] = getWorkspaceId @@CustomProperties['AgentVersion'] = ENV[@@EnvAgentVersion] - encodedAppInsightsKey = ENV[@@encodedAppInsightsKey] - decodedAppInsightsKey = Base64.decode64(encodedAppInsightsKey) - @@Tc = ApplicationInsights::TelemetryClient.new decodedAppInsightsKey + encodedAppInsightsKey = ENV[@@EnvApplicationInsightsKey] + if !encodedAppInsightsKey.nil? + decodedAppInsightsKey = Base64.decode64(encodedAppInsightsKey) + @@Tc = ApplicationInsights::TelemetryClient.new decodedAppInsightsKey + end rescue => errorStr $log.warn("Exception in AppInsightsUtility: initilizeUtility - error: #{errorStr}") end @@ -71,8 +71,10 @@ def initilizeutility() def sendHeartBeatEvent(pluginName, properties) begin eventName = pluginName + @@HeartBeat - @@Tc.track_event eventName , :properties => @@CustomProperties - @@Tc.flush + if !@@Tc.nil? + @@Tc.track_event eventName , :properties => @@CustomProperties + @@Tc.flush + end rescue =>errorStr $log.warn("Exception in AppInsightsUtility: sendHeartBeatEvent - error: #{errorStr}") end @@ -80,10 +82,12 @@ def sendHeartBeatEvent(pluginName, properties) def sendCustomEvent(pluginName, properties) begin - @@Tc.track_metric 'LastProcessedContainerInventoryCount', properties['ContainerCount'], - :kind => ApplicationInsights::Channel::Contracts::DataPointType::MEASUREMENT, - :properties => @@CustomProperties + if !@@Tc.nil? + @@Tc.track_metric 'LastProcessedContainerInventoryCount', properties['ContainerCount'], + :kind => ApplicationInsights::Channel::Contracts::DataPointType::MEASUREMENT, + :properties => @@CustomProperties @@Tc.flush + end rescue => errorStr $log.warn("Exception in AppInsightsUtility: sendCustomEvent - error: #{errorStr}") end @@ -92,20 +96,22 @@ def sendCustomEvent(pluginName, properties) def sendExceptionTelemetry(errorStr) begin if @@CustomProperties.empty? || @@CustomProperties.nil? - initilizeutility + initializeUtility + end + if !@@Tc.nil? + @@Tc.track_exception errorStr , :properties => @@CustomProperties + @@Tc.flush end - eventName = pluginName + @@Exception - @@Tc.track_exception errorStr , :properties => @@CustomProperties - @@Tc.flush rescue => errorStr $log.warn("Exception in AppInsightsUtility: sendExceptionTelemetry - error: #{errorStr}") end end + #Method to send heartbeat and container inventory count def sendTelemetry(pluginName, properties) begin if @@CustomProperties.empty? || @@CustomProperties.nil? - initilizeutility + initializeUtility end @@CustomProperties['Computer'] = properties['Computer'] sendHeartBeatEvent(pluginName, properties) diff --git a/source/code/plugin/DockerApiClient.rb b/source/code/plugin/DockerApiClient.rb index 5a73441da..b93411980 100644 --- a/source/code/plugin/DockerApiClient.rb +++ b/source/code/plugin/DockerApiClient.rb @@ -43,7 +43,7 @@ def getResponse(request, isMultiJson) return (isTimeOut)? nil : parseResponse(dockerResponse, isMultiJson) rescue => errorStr $log.warn("Socket call failed for request: #{request} error: #{errorStr} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") - ApplicationInsightsUtility.sendExceptionTelemetry(@@PluginName + '-' + errorStr) + ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) end end @@ -62,7 +62,7 @@ def parseResponse(dockerResponse, isMultiJson) end rescue => errorStr $log.warn("Json parsing for docker response failed: #{errorStr} , isMultiJson: #{isMultiJson} @ #{Time.now.utc.iso8601}") - ApplicationInsightsUtility.sendExceptionTelemetry(@@PluginName + '-' + errorStr) + ApplicationInsightsUtility.sendExceptionTelemetry(errorStr) end return parsedJsonResponse end From d992e9d478f556b0304cf238e825166028e01504 Mon Sep 17 00:00:00 2001 From: rashmy Date: Tue, 30 Oct 2018 21:30:05 -0700 Subject: [PATCH 33/45] fixes --- source/code/plugin/ApplicationInsightsUtility.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/code/plugin/ApplicationInsightsUtility.rb b/source/code/plugin/ApplicationInsightsUtility.rb index 6bd770949..ca9cffb4e 100644 --- a/source/code/plugin/ApplicationInsightsUtility.rb +++ b/source/code/plugin/ApplicationInsightsUtility.rb @@ -19,6 +19,7 @@ class ApplicationInsightsUtility @@EnvAgentVersion = 'AGENT_VERSION' @@EnvApplicationInsightsKey = 'APPLICATIONINSIGHTS_AUTH' @@CustomProperties = {} + @@Tc def initialize end @@ -116,6 +117,7 @@ def sendTelemetry(pluginName, properties) @@CustomProperties['Computer'] = properties['Computer'] sendHeartBeatEvent(pluginName, properties) sendCustomEvent(pluginName, properties) + $log.info("AppInsights Telemetry sent successfully") rescue => errorStr $log.warn("Exception in AppInsightsUtility: sendTelemetry - error: #{errorStr}") end From 17656aa6ec95d80a972002c63d55f4d403c2c0ad Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 31 Oct 2018 15:12:15 -0700 Subject: [PATCH 34/45] PR comments --- installer/conf/container.conf | 8 +------ .../code/plugin/ApplicationInsightsUtility.rb | 23 +++++++++---------- source/code/plugin/in_containerinventory.rb | 1 + 3 files changed, 13 insertions(+), 19 deletions(-) diff --git a/installer/conf/container.conf b/installer/conf/container.conf index 84be0d05f..798bd8eb6 100755 --- a/installer/conf/container.conf +++ b/installer/conf/container.conf @@ -7,12 +7,6 @@ bind 127.0.0.1 - - type oms_omi - object_name "Container" - interval 30s - - # Container inventory type containerinventory @@ -58,7 +52,7 @@ num_threads 5 buffer_chunk_limit 20m buffer_type file - buffer_path /var/opt/microsoft/omsagent/62a84175-b18d-48ff-8b42-0694dda7b8bf/state/state/out_oms_containerinventory*.buffer + buffer_path %STATE_DIR_WS%/out_oms_containerinventory*.buffer buffer_queue_limit 20 buffer_queue_full_action drop_oldest_chunk flush_interval 20s diff --git a/source/code/plugin/ApplicationInsightsUtility.rb b/source/code/plugin/ApplicationInsightsUtility.rb index ca9cffb4e..af139e0e3 100644 --- a/source/code/plugin/ApplicationInsightsUtility.rb +++ b/source/code/plugin/ApplicationInsightsUtility.rb @@ -37,16 +37,15 @@ def initializeUtility() @@CustomProperties["ClusterName"] = "" @@CustomProperties["Region"] = "" else - aksResourceId = ENV['AKS_RESOURCE_ID'] - @@CustomProperties["AKS_RESOURCE_ID"] = aksResourceId - begin - splitStrings = aksResourceId.split('/') - subscriptionId = splitStrings[2] - resourceGroupName = splitStrings[4] - clusterName = splitStrings[8] - rescue => errorStr - $log.warn("Exception in AppInsightsUtility: parsing AKS resourceId: #{aksResourceId}, error: #{errorStr}") - end + @@CustomProperties["AKS_RESOURCE_ID"] = resourceInfo + begin + splitStrings = resourceInfo.split('/') + subscriptionId = splitStrings[2] + resourceGroupName = splitStrings[4] + clusterName = splitStrings[8] + rescue => errorStr + $log.warn("Exception in AppInsightsUtility: parsing AKS resourceId: #{resourceInfo}, error: #{errorStr}") + end @@CustomProperties["ClusterType"] = @@AksClusterType @@CustomProperties["SubscriptionID"] = subscriptionId @@CustomProperties["ResourceGroupName"] = resourceGroupName @@ -69,7 +68,7 @@ def initializeUtility() end end - def sendHeartBeatEvent(pluginName, properties) + def sendHeartBeatEvent(pluginName) begin eventName = pluginName + @@HeartBeat if !@@Tc.nil? @@ -115,7 +114,7 @@ def sendTelemetry(pluginName, properties) initializeUtility end @@CustomProperties['Computer'] = properties['Computer'] - sendHeartBeatEvent(pluginName, properties) + sendHeartBeatEvent(pluginName) sendCustomEvent(pluginName, properties) $log.info("AppInsights Telemetry sent successfully") rescue => errorStr diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 48c870430..7614fa2b0 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -17,6 +17,7 @@ def initialize require 'json' require_relative 'DockerApiClient' require_relative 'ContainerInventoryState' + require_relative 'ApplicationInsightsUtility' require_relative 'omslog' end From 339bd21b9466cb35134725abffb75dc842da2784 Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 31 Oct 2018 15:48:20 -0700 Subject: [PATCH 35/45] changes --- source/code/plugin/ApplicationInsightsUtility.rb | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/source/code/plugin/ApplicationInsightsUtility.rb b/source/code/plugin/ApplicationInsightsUtility.rb index af139e0e3..14fc9f2f8 100644 --- a/source/code/plugin/ApplicationInsightsUtility.rb +++ b/source/code/plugin/ApplicationInsightsUtility.rb @@ -19,7 +19,7 @@ class ApplicationInsightsUtility @@EnvAgentVersion = 'AGENT_VERSION' @@EnvApplicationInsightsKey = 'APPLICATIONINSIGHTS_AUTH' @@CustomProperties = {} - @@Tc + @@Tc = nil def initialize end @@ -71,9 +71,10 @@ def initializeUtility() def sendHeartBeatEvent(pluginName) begin eventName = pluginName + @@HeartBeat - if !@@Tc.nil? + if !(@@Tc.nil?) @@Tc.track_event eventName , :properties => @@CustomProperties @@Tc.flush + $log.info("AppInsights Heartbeat Telemetry sent successfully") end rescue =>errorStr $log.warn("Exception in AppInsightsUtility: sendHeartBeatEvent - error: #{errorStr}") @@ -82,11 +83,12 @@ def sendHeartBeatEvent(pluginName) def sendCustomEvent(pluginName, properties) begin - if !@@Tc.nil? + if !(@@Tc.nil?) @@Tc.track_metric 'LastProcessedContainerInventoryCount', properties['ContainerCount'], :kind => ApplicationInsights::Channel::Contracts::DataPointType::MEASUREMENT, :properties => @@CustomProperties - @@Tc.flush + @@Tc.flush + $log.info("AppInsights Container Count Telemetry sent successfully") end rescue => errorStr $log.warn("Exception in AppInsightsUtility: sendCustomEvent - error: #{errorStr}") @@ -98,9 +100,10 @@ def sendExceptionTelemetry(errorStr) if @@CustomProperties.empty? || @@CustomProperties.nil? initializeUtility end - if !@@Tc.nil? + if !(@@Tc.nil?) @@Tc.track_exception errorStr , :properties => @@CustomProperties @@Tc.flush + $log.info("AppInsights Exception Telemetry sent successfully") end rescue => errorStr $log.warn("Exception in AppInsightsUtility: sendExceptionTelemetry - error: #{errorStr}") @@ -116,7 +119,6 @@ def sendTelemetry(pluginName, properties) @@CustomProperties['Computer'] = properties['Computer'] sendHeartBeatEvent(pluginName) sendCustomEvent(pluginName, properties) - $log.info("AppInsights Telemetry sent successfully") rescue => errorStr $log.warn("Exception in AppInsightsUtility: sendTelemetry - error: #{errorStr}") end From fcfc437ca9d9cd88ca8e066eeb2551c20419447a Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 31 Oct 2018 16:33:17 -0700 Subject: [PATCH 36/45] updating the ownership --- installer/datafiles/base_container.data | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/installer/datafiles/base_container.data b/installer/datafiles/base_container.data index 610e642f7..3ed69ec7e 100644 --- a/installer/datafiles/base_container.data +++ b/installer/datafiles/base_container.data @@ -74,7 +74,7 @@ MAINTAINER: 'Microsoft Corporation' /var/opt/microsoft; 755; root; root; sysdir /var/opt/microsoft/docker-cimprov; 755; root; root /var/opt/microsoft/docker-cimprov/state; 755; root; root -/var/opt/microsoft/docker-cimprov/state/ContainerInventory; 755; omsagent; omsagent +/var/opt/microsoft/docker-cimprov/state/ContainerInventory; 755; root; root /var/opt/microsoft/docker-cimprov/log; 755; root; root /opt/td-agent-bit; 755; root; root;sysdir @@ -89,6 +89,9 @@ WriteInstallInfo() { } WriteInstallInfo +#Make omsagent owner for ContainerInventory directory. This is needed for ruby plugin to have access +chown omsagent:omsagent /var/opt/microsoft/docker-cimprov/state/ContainerInventory + # Get the state file in place with proper permissions touch /var/opt/microsoft/docker-cimprov/state/LastEventQueryTime.txt chmod 644 /var/opt/microsoft/docker-cimprov/state/LastEventQueryTime.txt From d434d9846ea236e07951403cc85c10a5f016e1b1 Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 31 Oct 2018 17:28:07 -0700 Subject: [PATCH 37/45] changes --- installer/datafiles/base_container.data | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/installer/datafiles/base_container.data b/installer/datafiles/base_container.data index 3ed69ec7e..92c7fc6cb 100644 --- a/installer/datafiles/base_container.data +++ b/installer/datafiles/base_container.data @@ -37,6 +37,14 @@ MAINTAINER: 'Microsoft Corporation' /opt/microsoft/omsagent/plugin/in_kube_services.rb; source/code/plugin/in_kube_services.rb; 644; root; root /opt/microsoft/omsagent/plugin/in_kube_nodes.rb; source/code/plugin/in_kube_nodes.rb; 644; root; root +/opt/microsoft/omsagent/plugin/ApplicationInsightsUtility.rb source/code/plugin/ApplicationInsightsUtility.rb 644; root; root +/opt/microsoft/omsagent/plugin/ContainerInventoryState.rb source/code/plugin/ContainerInventoryState.rb 644; root; root +/opt/microsoft/omsagent/plugin/DockerApiClient.rb source/code/plugin/DockerApiClient.rb 644; root; root +/opt/microsoft/omsagent/plugin/DockerApiRestHelper.rb source/code/plugin/DockerApiRestHelper.rb 644; root; root +/opt/microsoft/omsagent/plugin/in_containerinventory.rb source/code/plugin/in_containerinventory.rb 644; root; root +/opt/microsoft/omsagent/plugin/lib source/code/plugin/lib 644; root; root + + /opt/td-agent-bit/bin/out_oms.so; intermediate/${{BUILD_CONFIGURATION}}/out_oms.so; 755; root; root /etc/opt/microsoft/docker-cimprov/td-agent-bit.conf; installer/conf/td-agent-bit.conf; 644; root; root /etc/opt/microsoft/docker-cimprov/out_oms.conf; installer/conf/out_oms.conf; 644; root; root From b049ba0160f6d4102d3be8b7b34325f9b3e545a4 Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 31 Oct 2018 17:35:53 -0700 Subject: [PATCH 38/45] changes --- installer/datafiles/base_container.data | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/installer/datafiles/base_container.data b/installer/datafiles/base_container.data index 92c7fc6cb..7a8571270 100644 --- a/installer/datafiles/base_container.data +++ b/installer/datafiles/base_container.data @@ -37,12 +37,12 @@ MAINTAINER: 'Microsoft Corporation' /opt/microsoft/omsagent/plugin/in_kube_services.rb; source/code/plugin/in_kube_services.rb; 644; root; root /opt/microsoft/omsagent/plugin/in_kube_nodes.rb; source/code/plugin/in_kube_nodes.rb; 644; root; root -/opt/microsoft/omsagent/plugin/ApplicationInsightsUtility.rb source/code/plugin/ApplicationInsightsUtility.rb 644; root; root -/opt/microsoft/omsagent/plugin/ContainerInventoryState.rb source/code/plugin/ContainerInventoryState.rb 644; root; root -/opt/microsoft/omsagent/plugin/DockerApiClient.rb source/code/plugin/DockerApiClient.rb 644; root; root -/opt/microsoft/omsagent/plugin/DockerApiRestHelper.rb source/code/plugin/DockerApiRestHelper.rb 644; root; root -/opt/microsoft/omsagent/plugin/in_containerinventory.rb source/code/plugin/in_containerinventory.rb 644; root; root -/opt/microsoft/omsagent/plugin/lib source/code/plugin/lib 644; root; root +/opt/microsoft/omsagent/plugin/ApplicationInsightsUtility.rb; source/code/plugin/ApplicationInsightsUtility.rb; 644; root; root +/opt/microsoft/omsagent/plugin/ContainerInventoryState.rb; source/code/plugin/ContainerInventoryState.rb; 644; root; root +/opt/microsoft/omsagent/plugin/DockerApiClient.rb; source/code/plugin/DockerApiClient.rb; 644; root; root +/opt/microsoft/omsagent/plugin/DockerApiRestHelper.rb; source/code/plugin/DockerApiRestHelper.rb; 644; root; root +/opt/microsoft/omsagent/plugin/in_containerinventory.rb; source/code/plugin/in_containerinventory.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib; source/code/plugin/lib; 644; root; root /opt/td-agent-bit/bin/out_oms.so; intermediate/${{BUILD_CONFIGURATION}}/out_oms.so; 755; root; root From e9111a978313d64ebabb854c3d88f56efaa629a5 Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 31 Oct 2018 18:17:55 -0700 Subject: [PATCH 39/45] changes to container data --- installer/datafiles/base_container.data | 58 ++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 7 deletions(-) diff --git a/installer/datafiles/base_container.data b/installer/datafiles/base_container.data index 7a8571270..69a32fa28 100644 --- a/installer/datafiles/base_container.data +++ b/installer/datafiles/base_container.data @@ -37,13 +37,57 @@ MAINTAINER: 'Microsoft Corporation' /opt/microsoft/omsagent/plugin/in_kube_services.rb; source/code/plugin/in_kube_services.rb; 644; root; root /opt/microsoft/omsagent/plugin/in_kube_nodes.rb; source/code/plugin/in_kube_nodes.rb; 644; root; root -/opt/microsoft/omsagent/plugin/ApplicationInsightsUtility.rb; source/code/plugin/ApplicationInsightsUtility.rb; 644; root; root -/opt/microsoft/omsagent/plugin/ContainerInventoryState.rb; source/code/plugin/ContainerInventoryState.rb; 644; root; root -/opt/microsoft/omsagent/plugin/DockerApiClient.rb; source/code/plugin/DockerApiClient.rb; 644; root; root -/opt/microsoft/omsagent/plugin/DockerApiRestHelper.rb; source/code/plugin/DockerApiRestHelper.rb; 644; root; root -/opt/microsoft/omsagent/plugin/in_containerinventory.rb; source/code/plugin/in_containerinventory.rb; 644; root; root -/opt/microsoft/omsagent/plugin/lib; source/code/plugin/lib; 644; root; root - +/opt/microsoft/omsagent/plugin/ApplicationInsightsUtility.rb; source/code/plugin/ApplicationInsightsUtility.rb; 644; root; root +/opt/microsoft/omsagent/plugin/ContainerInventoryState.rb; source/code/plugin/ContainerInventoryState.rb; 644; root; root +/opt/microsoft/omsagent/plugin/DockerApiClient.rb; source/code/plugin/DockerApiClient.rb; 644; root; root +/opt/microsoft/omsagent/plugin/DockerApiRestHelper.rb; source/code/plugin/DockerApiRestHelper.rb; 644; root; root +/opt/microsoft/omsagent/plugin/in_containerinventory.rb; source/code/plugin/in_containerinventory.rb; 644; root; root + +# Files for application insights library +/opt/microsoft/omsagent/plugin/lib/application_insights/version.rb; source/code/plugin/lib/application_insights/version.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/rack/track_request.rb; source/code/plugin/lib/application_insights/rack/track_request.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/unhandled_exception.rb; source/code/plugin/lib/application_insights/unhandled_exception.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/telemetry_client.rb; source/code/plugin/lib/application_insights/telemetry_client.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/queue_base.rb; source/code/plugin/lib/application_insights/channel/queue_base.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/asynchronous_queue.rb; source/code/plugin/lib/application_insights/channel/asynchronous_queue.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/synchronous_sender.rb; source/code/plugin/lib/application_insights/channel/synchronous_sender.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/data_point_type.rb; source/code/plugin/lib/application_insights/channel/contracts/data_point_type.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/data_point.rb; source/code/plugin/lib/application_insights/channel/contracts/data_point.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/stack_frame.rb; source/code/plugin/lib/application_insights/channel/contracts/stack_frame.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/request_data.rb; source/code/plugin/lib/application_insights/channel/contracts/request_data.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/session.rb; source/code/plugin/lib/application_insights/channel/contracts/session.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/page_view_data.rb; source/code/plugin/lib/application_insights/channel/contracts/page_view_data.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/remote_dependency_data.rb; source/code/plugin/lib/application_insights/channel/contracts/remote_dependency_data.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/exception_data.rb; source/code/plugin/lib/application_insights/channel/contracts/exception_data.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/location.rb; source/code/plugin/lib/application_insights/channel/contracts/location.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/operation.rb; source/code/plugin/lib/application_insights/channel/contracts/operation.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/data.rb; source/code/plugin/lib/application_insights/channel/contracts/data.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/event_data.rb; source/code/plugin/lib/application_insights/channel/contracts/event_data.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/metric_data.rb; source/code/plugin/lib/application_insights/channel/contracts/metric_data.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/device.rb; source/code/plugin/lib/application_insights/channel/contracts/device.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/message_data.rb; source/code/plugin/lib/application_insights/channel/contracts/message_data.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/dependency_source_type.rb; source/code/plugin/lib/application_insights/channel/contracts/dependency_source_type.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/user.rb; source/code/plugin/lib/application_insights/channel/contracts/user.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/severity_level.rb; source/code/plugin/lib/application_insights/channel/contracts/severity_level.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/application.rb; source/code/plugin/lib/application_insights/channel/contracts/application.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/dependency_kind.rb; source/code/plugin/lib/application_insights/channel/contracts/dependency_kind.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/cloud.rb; source/code/plugin/lib/application_insights/channel/contracts/cloud.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/envelope.rb; source/code/plugin/lib/application_insights/channel/contracts/envelope.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/json_serializable.rb; source/code/plugin/lib/application_insights/channel/contracts/json_serializable.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/domain.rb; source/code/plugin/lib/application_insights/channel/contracts/domain.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/base.rb; source/code/plugin/lib/application_insights/channel/contracts/base.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/reopenings.rb; source/code/plugin/lib/application_insights/channel/contracts/reopenings.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/page_view_perf_data.rb; source/code/plugin/lib/application_insights/channel/contracts/page_view_perf_data.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/internal.rb; source/code/plugin/lib/application_insights/channel/contracts/internal.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/availability_data.rb; source/code/plugin/lib/application_insights/channel/contracts/availability_data.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts/exception_details.rb; source/code/plugin/lib/application_insights/channel/contracts/exception_details.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/synchronous_queue.rb; source/code/plugin/lib/application_insights/channel/synchronous_queue.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/sender_base.rb; source/code/plugin/lib/application_insights/channel/sender_base.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/telemetry_context.rb; source/code/plugin/lib/application_insights/channel/telemetry_context.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/asynchronous_sender.rb; source/code/plugin/lib/application_insights/channel/asynchronous_sender.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/telemetry_channel.rb; source/code/plugin/lib/application_insights/channel/telemetry_channel.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/event.rb; source/code/plugin/lib/application_insights/channel/event.rb; 644; root; root +/opt/microsoft/omsagent/plugin/lib/application_insights.rb; source/code/plugin/lib/application_insights.rb; 644; root; root /opt/td-agent-bit/bin/out_oms.so; intermediate/${{BUILD_CONFIGURATION}}/out_oms.so; 755; root; root /etc/opt/microsoft/docker-cimprov/td-agent-bit.conf; installer/conf/td-agent-bit.conf; 644; root; root From 0edb539545de638d9cd29aa2e3109b170f2f0e47 Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 31 Oct 2018 18:20:56 -0700 Subject: [PATCH 40/45] removing comment --- installer/datafiles/base_container.data | 1 - 1 file changed, 1 deletion(-) diff --git a/installer/datafiles/base_container.data b/installer/datafiles/base_container.data index 69a32fa28..1d81fb346 100644 --- a/installer/datafiles/base_container.data +++ b/installer/datafiles/base_container.data @@ -43,7 +43,6 @@ MAINTAINER: 'Microsoft Corporation' /opt/microsoft/omsagent/plugin/DockerApiRestHelper.rb; source/code/plugin/DockerApiRestHelper.rb; 644; root; root /opt/microsoft/omsagent/plugin/in_containerinventory.rb; source/code/plugin/in_containerinventory.rb; 644; root; root -# Files for application insights library /opt/microsoft/omsagent/plugin/lib/application_insights/version.rb; source/code/plugin/lib/application_insights/version.rb; 644; root; root /opt/microsoft/omsagent/plugin/lib/application_insights/rack/track_request.rb; source/code/plugin/lib/application_insights/rack/track_request.rb; 644; root; root /opt/microsoft/omsagent/plugin/lib/application_insights/unhandled_exception.rb; source/code/plugin/lib/application_insights/unhandled_exception.rb; 644; root; root From abf1a67f8e49aeff8309d9b52f57009f660bfb7c Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 31 Oct 2018 18:27:15 -0700 Subject: [PATCH 41/45] changes --- installer/datafiles/base_container.data | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/installer/datafiles/base_container.data b/installer/datafiles/base_container.data index 1d81fb346..7181929e2 100644 --- a/installer/datafiles/base_container.data +++ b/installer/datafiles/base_container.data @@ -131,6 +131,12 @@ MAINTAINER: 'Microsoft Corporation' /opt/td-agent-bit; 755; root; root;sysdir /opt/td-agent-bit/bin; 755; root; root;sysdir +/opt/microsoft/omsagent/plugin/lib; 755; root; root; sysdir +/opt/microsoft/omsagent/plugin/lib/application_insights; 755; root; root; sysdir +/opt/microsoft/omsagent/plugin/lib/application_insights/channel; 755; root; root; sysdir +/opt/microsoft/omsagent/plugin/lib/application_insights/channel/contracts; 755; root; root; sysdir +/opt/microsoft/omsagent/plugin/lib/application_insights/rack; 755; root; root; sysdir + %Dependencies %Postinstall_10 From 2c4ddff5c0504dab90b3f32e75ed6c2ec02c75aa Mon Sep 17 00:00:00 2001 From: rashmy Date: Wed, 31 Oct 2018 18:55:06 -0700 Subject: [PATCH 42/45] adding collection time --- source/code/plugin/in_containerinventory.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 7614fa2b0..17bd78882 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -183,6 +183,7 @@ def enumerate nameMap = DockerApiClient.getImageIdMap containerIds.each do |containerId| inspectedContainer = {} + inspectedContainer['CollectionTime'] = batchTime #This is the time that is mapped to become TimeGenerated inspectedContainer = inspectContainer(containerId, nameMap) inspectedContainer['Computer'] = hostname containerInventory.push inspectedContainer From 0d3395b9e70ac0512feff5dc5f0fc2496d3f3714 Mon Sep 17 00:00:00 2001 From: rashmy Date: Thu, 1 Nov 2018 18:48:30 -0700 Subject: [PATCH 43/45] bug fix --- source/code/plugin/in_containerinventory.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index 17bd78882..ed41ff136 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -183,9 +183,9 @@ def enumerate nameMap = DockerApiClient.getImageIdMap containerIds.each do |containerId| inspectedContainer = {} - inspectedContainer['CollectionTime'] = batchTime #This is the time that is mapped to become TimeGenerated inspectedContainer = inspectContainer(containerId, nameMap) inspectedContainer['Computer'] = hostname + inspectedContainer['CollectionTime'] = batchTime #This is the time that is mapped to become TimeGenerated containerInventory.push inspectedContainer ContainerInventoryState.writeContainerState(inspectedContainer) end From 273af3b187c7bc7084ec4366f82e8a3ca71ed3da Mon Sep 17 00:00:00 2001 From: rashmy Date: Sat, 3 Nov 2018 12:40:22 -0700 Subject: [PATCH 44/45] env string truncation --- source/code/plugin/in_containerinventory.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index ed41ff136..bd2909618 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -56,7 +56,17 @@ def obtainContainerConfig(instance, container) envValue = configValue['Env'] envValueString = (envValue.nil?) ? "" : envValue.to_s - instance['EnvironmentVar'] = envValueString + # Restricting the ENV string value to 200kb since the size of this string can go very high + if envValueString.length > 200000 + envValueStringTruncated = envValueString.slice(0..200000) + lastIndex = envValueStringTruncated.rindex("\", ") + if !lastIndex.nil? + envValueStringTruncated = envValueStringTruncated.slice(0..lastIndex) + "]" + end + instance['EnvironmentVar'] = envValueStringTruncated + else + instance['EnvironmentVar'] = envValueString + end cmdValue = configValue['Cmd'] cmdValueString = (cmdValue.nil?) ? "" : cmdValue.to_s From 3b74ca7e31df42f1b72edb496e10127110e8e9dc Mon Sep 17 00:00:00 2001 From: rashmy Date: Mon, 5 Nov 2018 12:37:18 -0800 Subject: [PATCH 45/45] changes for acs-engine test --- source/code/plugin/in_containerinventory.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/code/plugin/in_containerinventory.rb b/source/code/plugin/in_containerinventory.rb index bd2909618..43811e1e1 100644 --- a/source/code/plugin/in_containerinventory.rb +++ b/source/code/plugin/in_containerinventory.rb @@ -221,6 +221,10 @@ def enumerate eventStream.add(emitTime, wrapper) if wrapper end router.emit_stream(@tag, eventStream) if eventStream + @@istestvar = ENV['ISTEST'] + if (!@@istestvar.nil? && !@@istestvar.empty? && @@istestvar.casecmp('true') == 0 && eventStream.count > 0) + $log.info("containerInventoryEmitStreamSuccess @ #{Time.now.utc.iso8601}") + end timeDifference = (DateTime.now.to_time.to_i - @@telemetryTimeTracker).abs timeDifferenceInMinutes = timeDifference/60 if (timeDifferenceInMinutes >= 5)