Skip to content

Fix Unscheduled Pod bug, remove excess telemetry #218

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
r-dilip merged 3 commits into from
May 31, 2019
Merged

Fix Unscheduled Pod bug, remove excess telemetry #218

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8a766d6
Fix Unscheduled Pod bug, remove excess telemetry
r-dilip May 28, 2019
b1fbc37
Send Success Telemetry only once after startup for a node in a cluste…
r-dilip May 29, 2019
8ab08b8
Sending telemetry for successful push to MDM every hour
r-dilip May 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 67 additions & 61 deletions source/code/plugin/filter_inventory2mdm.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ module Fluent

class Inventory2MdmFilter < Filter
Fluent::Plugin.register_filter('filter_inventory2mdm', self)

config_param :enable_log, :integer, :default => 0
config_param :log_path, :string, :default => '/var/opt/microsoft/docker-cimprov/log/filter_inventory2mdm.log'
config_param :custom_metrics_azure_regions, :string

@@node_count_metric_name = 'nodesCount'
@@pod_count_metric_name = 'podCount'
@@pod_inventory_tag = 'mdm.kubepodinventory'
Expand All @@ -23,63 +23,63 @@ class Inventory2MdmFilter < Filter
@@node_status_not_ready = 'NotReady'

@@node_inventory_custom_metrics_template = '
{
"time": "%{timestamp}",
"data": {
"baseData": {
"metric": "%{metricName}",
"namespace": "insights.container/nodes",
"dimNames": [
{
"time": "%{timestamp}",
"data": {
"baseData": {
"metric": "%{metricName}",
"namespace": "insights.container/nodes",
"dimNames": [
"status"
],
"series": [
{
"dimValues": [
],
"series": [
{
"dimValues": [
"%{statusValue}"
],
],
"min": %{node_status_count},
"max": %{node_status_count},
"sum": %{node_status_count},
"max": %{node_status_count},
"sum": %{node_status_count},
"count": 1
}
]
}
}
}
]
}
}
}'

@@pod_inventory_custom_metrics_template = '
{
"time": "%{timestamp}",
"data": {
"baseData": {
"metric": "%{metricName}",
"namespace": "insights.container/pods",
"dimNames": [
"phase",
"Kubernetes namespace",
"node",
{
"time": "%{timestamp}",
"data": {
"baseData": {
"metric": "%{metricName}",
"namespace": "insights.container/pods",
"dimNames": [
"phase",
"Kubernetes namespace",
"node",
"controllerName"
],
"series": [
{
"dimValues": [
"%{phaseDimValue}",
"%{namespaceDimValue}",
"%{nodeDimValue}",
],
"series": [
{
"dimValues": [
"%{phaseDimValue}",
"%{namespaceDimValue}",
"%{nodeDimValue}",
"%{controllerNameDimValue}"
],
],
"min": %{podCountMetricValue},
"max": %{podCountMetricValue},
"sum": %{podCountMetricValue},
"count": 1
}
]
}
}
"max": %{podCountMetricValue},
"sum": %{podCountMetricValue},
"count": 1
}
]
}
}
}'

@@pod_phase_values = ['Running', 'Pending', 'Succeeded', 'Failed', 'Unknown']

@process_incoming_stream = true

def initialize
Expand All @@ -89,7 +89,7 @@ def initialize
def configure(conf)
super
@log = nil

if @enable_log
@log = Logger.new(@log_path, 1, 5000000)
@log.debug {'Starting filter_inventory2mdm plugin'}
Expand All @@ -105,15 +105,15 @@ def start
def shutdown
super
end

def process_node_inventory_records(es)
timestamp = DateTime.now

begin
node_ready_count = 0
node_not_ready_count = 0
records = []

es.each{|time,record|
begin
timestamp = record['DataItems'][0]['CollectionTime']
Expand All @@ -129,15 +129,15 @@ def process_node_inventory_records(es)

ready_record = @@node_inventory_custom_metrics_template % {
timestamp: timestamp,
metricName: @@node_count_metric_name,
metricName: @@node_count_metric_name,
statusValue: @@node_status_ready,
node_status_count: node_ready_count
}
records.push(JSON.parse(ready_record))

not_ready_record = @@node_inventory_custom_metrics_template % {
timestamp: timestamp,
metricName: @@node_count_metric_name,
metricName: @@node_count_metric_name,
statusValue: @@node_status_not_ready,
node_status_count: node_not_ready_count
}
Expand All @@ -164,7 +164,7 @@ def process_pod_inventory_records(es)
record_count += 1
timestamp = record['DataItems'][0]['CollectionTime']
podUid = record['DataItems'][0]['PodUid']

if podUids.key?(podUid)
#@log.info "pod with #{podUid} already counted"
next
Expand All @@ -176,6 +176,12 @@ def process_pod_inventory_records(es)
podControllerNameDimValue = record['DataItems'][0]['ControllerName']
podNodeDimValue = record['DataItems'][0]['Computer']

if podNodeDimValue.empty? && podPhaseDimValue.downcase == 'pending'
podNodeDimValue = 'unscheduled'
elsif podNodeDimValue.empty?
podNodeDimValue = 'unknown'
end

# group by distinct dimension values
pod_key = [podNodeDimValue, podNamespaceDimValue, podControllerNameDimValue, podPhaseDimValue].join('~~')

Expand All @@ -197,7 +203,7 @@ def process_pod_inventory_records(es)
pod_count = 1
pod_count_hash[pod_key] = pod_count
end

# Collect all possible combinations of dimension values other than pod phase
key_without_phase_dim_value = [podNodeDimValue, podNamespaceDimValue, podControllerNameDimValue].join('~~')
if no_phase_dim_values_hash.key?(key_without_phase_dim_value)
Expand Down Expand Up @@ -237,9 +243,9 @@ def process_pod_inventory_records(es)
timestamp: timestamp,
metricName: @@pod_count_metric_name,
phaseDimValue: podPhaseDimValue,
namespaceDimValue: podNamespaceDimValue,
nodeDimValue: podNodeDimValue,
controllerNameDimValue: podControllerNameDimValue,
namespaceDimValue: podNamespaceDimValue,
nodeDimValue: podNodeDimValue,
controllerNameDimValue: podControllerNameDimValue,
podCountMetricValue: value
}
records.push(JSON.parse(record))
Expand All @@ -265,11 +271,11 @@ def filter_stream(tag, es)
elsif tag.downcase.start_with?(@@pod_inventory_tag)
@log.info 'Processing POD inventory records in filter plugin to send to MDM'
filtered_records, time = process_pod_inventory_records(es)
else
else
filtered_records = []
end
end
filtered_records.each {|filtered_record|
filtered_records.each {|filtered_record|
new_es.add(time, filtered_record) if filtered_record
} if filtered_records
rescue => e
Expand Down
7 changes: 6 additions & 1 deletion source/code/plugin/out_mdm.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ def initialize
@last_post_attempt_time = Time.now
@first_post_attempt_made = false
@can_send_data_to_mdm = true
@last_telemetry_sent_time = nil
end

def configure(conf)
Expand Down Expand Up @@ -156,7 +157,11 @@ def send_to_mdm(post_body)
response = @http_client.request(request)
response.value # this throws for non 200 HTTP response code
@log.info "HTTP Post Response Code : #{response.code}"
ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMSendSuccessful", {})
if @last_telemetry_sent_time.nil? || @last_telemetry_sent_time + 60 * 60 < Time.now
Copy link
Member

@vishiy vishiy May 30, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should this be '>' ?

ApplicationInsightsUtility.sendCustomEvent("AKSCustomMetricsMDMSendSuccessful", {})
@last_telemetry_sent_time = Time.now
end

rescue Net::HTTPServerException => e
@log.info "Failed to Post Metrics to MDM : #{e} Response: #{response}"
@log.debug_backtrace(e.backtrace)
Expand Down