Merge ci_feature into ci_feature_prod

.gitignore

@@ -8,3 +8,6 @@
 
 /test/code/providers/TestScriptPath.h
 /test/code/providers/providertestutils.cpp
+source/code/go/src/plugins/profiling
+.vscode/launch.json
+source/code/go/src/plugins/vendor/

README.md

@@ -9,6 +9,14 @@ additional questions or comments.
 
 ## Release History
 
+### 10/16/2018 - Version microsoft/oms:ciprod10162018
+- Fix for containerID being 00000-00000-00000
+- Move from fluentD to fluentbit for container log collection
+- Seg fault fixes in json parsing for container inventory & container image inventory
+- Telemetry enablement
+- Remove ContainerPerf, ContainerServiceLog, ContainerProcess fluentd-->OMI workflows
+- Update log level for all fluentD based workflows 
+
 ### 7/31/2018 - Version microsoft/oms:ciprod07312018
 - Changes for node lost scenario (roll-up pod & container statuses as Unknown)
 - Discover unscheduled pods
@@ -32,6 +40,4 @@ additional questions or comments.
 - Kubernetes RBAC enablement
 - Latest released omsagent (1.6.0-42)
 - Bug fix so that we do not collect kube-system namespace container logs when kube api calls fail occasionally (Bug #215107)
-- .yaml changes (for RBAC)
-
-
+- .yaml changes (for RBAC)

installer/conf/container.conf

@@ -7,19 +7,6 @@
 	bind 127.0.0.1
 </source>
 
-# Filter container logs
-<filter docker.**>
-	type filter_docker_log
-	log_path "/var/opt/microsoft/omsagent/log/filter_docker_log.txt"
-</filter>
-
-# Container perf
-<source>
-	type oms_omi
-	object_name "Container"
-	interval 30s
-</source>
-
 # Container inventory
 <source>
 	type omi
@@ -40,28 +27,6 @@
 	]
 </source>
 
-# Container service log
-<source>
-	type omi
-	run_interval 60s
-	tag oms.container.servicelog
-	items [
-		["root/cimv2","Container_DaemonEvent"]
-	]
-</source>
-
-# Container log
-# Example line which matches the format:
-# {"log"=>"Test 9th January\n", "stream"=>"stdout", "time"=>"2018-01-09T23:14:39.273429353Z", "ContainerID"=>"ee1ec26aa974af81b21fff24cef8ec78bf7ac1558b5de6f1eb1a5b28ecd6d559", "Image"=>"ubuntu", "Name"=>"determined_wilson", "SourceSystem"=>"Containers"}
-# NOTE: The LogEntryTimeStamp is just being appended in the begining of the LogEntry field. This is the actual time the log was generated and the TimeGenerated field in Kusto is different
-<source>
-	type containerlog_sudo_tail
-	pos_file /var/opt/microsoft/docker-cimprov/state/ContainerLogFile.pos.log
-	tag oms.container.log
-	format /\"log\"=>\"(?<LogEntry>.*)", \"stream\"=>\"(?<LogEntrySource>.*)", \"time\"=>\"(?<LogEntryTimeStamp>.*)", \"ContainerID\"=>\"(?<Id>.*)", \"Image\"=>\"(?<Image>.*)", \"Name\"=>\"(?<Name>.*)", \"SourceSystem\"=>\"(?<SourceSystem>.*)"}/
-	run_interval 60s
-</source>
-
 # Container host inventory
 <source>
 	type omi
@@ -72,16 +37,6 @@
 	]
 </source>
 
-# Container processes
-<source>
-	type omi
-	run_interval 60s
-	tag oms.api.ContainerProcess
-	items [
-		["root/cimv2","Container_Process"]
-	]
-</source>
-
 #cadvisor perf
 <source>
 	type cadvisorperf
@@ -91,15 +46,10 @@
 </source>
 
 # Filter for correct format to endpoint
-<filter oms.container.containerinventory oms.container.imageinventory oms.container.servicelog>
+<filter oms.container.containerinventory oms.container.imageinventory>
 	type filter_container
 </filter>
 
-# Seperate filter for container log
-<filter oms.container.log>  
-	type filter_container_log  
-</filter> 
-
 <match oms.api.ContainerNodeInventory**>
   type out_oms_api
   log_level debug
@@ -113,22 +63,9 @@
   max_retry_wait 9m
 </match>
 
-<match oms.api.ContainerProcess**>
-  type out_oms_api
-  log_level debug
-  buffer_chunk_limit 20m
-  buffer_type file
-  buffer_path %STATE_DIR_WS%/out_oms_containerprocess*.buffer
-  buffer_queue_limit 20
-  flush_interval 20s
-  retry_limit 10
-  retry_wait 15s
-  max_retry_wait 9m
-</match>
-
 <match oms.container.containerinventory**>	
   type out_oms
-  log_level info
+  log_level debug
   buffer_chunk_limit 20m
   buffer_type file
   buffer_path %STATE_DIR_WS%/out_oms_containerinventory*.buffer
@@ -140,39 +77,11 @@
 </match>
 
 <match oms.container.imageinventory**>
-  type out_oms
-  log_level info
-  buffer_chunk_limit 20m
-  buffer_type file
-  buffer_path %STATE_DIR_WS%/out_oms_imageinventory*.buffer
-  buffer_queue_limit 20
-  flush_interval 20s
-  retry_limit 10
-  retry_wait 15s
-  max_retry_wait 9m
-</match>
-
-<match oms.container.log>
   type out_oms
   log_level debug
-  num_threads 5
   buffer_chunk_limit 20m
   buffer_type file
-  buffer_path %STATE_DIR_WS%/out_oms_log*.buffer
-  buffer_queue_limit 20
-  buffer_queue_full_action drop_oldest_chunk
-  flush_interval 20s
-  retry_limit 10
-  retry_wait 15s
-  max_retry_wait 9m
-</match>
-
-<match oms.container.servicelog**>
-  type out_oms
-  log_level info
-  buffer_chunk_limit 20m
-  buffer_type file
-  buffer_path %STATE_DIR_WS%/out_oms_servicelog*.buffer
+  buffer_path %STATE_DIR_WS%/out_oms_imageinventory*.buffer
   buffer_queue_limit 20
   flush_interval 20s
   retry_limit 10

installer/conf/out_oms.conf

@@ -0,0 +1,6 @@
+omsadmin_conf_path=/etc/opt/microsoft/omsagent/conf/omsadmin.conf
+cert_file_path=/etc/opt/microsoft/omsagent/certs/oms.crt
+key_file_path=/etc/opt/microsoft/omsagent/certs/oms.key
+container_host_file_path=/var/opt/microsoft/docker-cimprov/state/containerhostname
+container_inventory_refresh_interval=60
+kube_system_containers_refresh_interval=300

installer/conf/td-agent-bit.conf

@@ -0,0 +1,22 @@
+[SERVICE]
+    Flush         30
+    Log_Level     info
+    Parsers_File  /etc/td-agent-bit/parsers.conf
+    Log_File      /var/opt/microsoft/docker-cimprov/log/fluent-bit.log
+
+[INPUT]
+    Name tail
+    Tag oms.container.log.*
+    Path /var/log/containers/*.log
+    DB /var/opt/microsoft/docker-cimprov/state/fblogs.db
+    Parser docker
+    Mem_Buf_Limit 30m
+    Path_Key filepath
+    Skip_Long_Lines On
+
+[OUTPUT]
+    Name                            oms
+    EnableTelemetry                 true
+    TelemetryPushIntervalSeconds    300
+    Match                           oms.container.log.*
+    AgentVersion                    ciprod10162018

installer/datafiles/base_container.data

@@ -23,14 +23,11 @@ MAINTAINER:              'Microsoft Corporation'
 
 /opt/microsoft/omsagent/plugin/filter_docker_log.rb;			source/code/plugin/filter_docker_log.rb;			644; root; root
 /opt/microsoft/omsagent/plugin/filter_container.rb;			    source/code/plugin/filter_container.rb;				644; root; root
-/opt/microsoft/omsagent/plugin/filter_container_log.rb;			source/code/plugin/filter_container_log.rb;		    644; root; root
 
 /opt/microsoft/omsagent/plugin/in_kube_podinventory.rb;			source/code/plugin/in_kube_podinventory.rb;			644; root; root
 /opt/microsoft/omsagent/plugin/in_kube_events.rb;			    source/code/plugin/in_kube_events.rb;				644; root; root
 /opt/microsoft/omsagent/plugin/in_kube_logs.rb;                 source/code/plugin/in_kube_logs.rb;                 644; root; root
 /opt/microsoft/omsagent/plugin/KubernetesApiClient.rb;			source/code/plugin/KubernetesApiClient.rb;			644; root; root
-/opt/microsoft/omsagent/plugin/in_containerlog_sudo_tail.rb;		source/code/plugin/in_containerlog_sudo_tail.rb;	644; root; root
-/opt/microsoft/omsagent/plugin/containerlogtailfilereader.rb;		source/code/plugin/containerlogtailfilereader.rb;	744; root; root 
 
 /etc/opt/microsoft/docker-cimprov/container.conf;			    installer/conf/container.conf;                      644; root; root
 
@@ -40,7 +37,9 @@ MAINTAINER:              'Microsoft Corporation'
 /opt/microsoft/omsagent/plugin/in_kube_services.rb;			        source/code/plugin/in_kube_services.rb;			    	644; root; root
 /opt/microsoft/omsagent/plugin/in_kube_nodes.rb;			        source/code/plugin/in_kube_nodes.rb;			    	644; root; root
 
-
+/opt/td-agent-bit/bin/out_oms.so;                                intermediate/${{BUILD_CONFIGURATION}}/out_oms.so;  	755; root; root
+/etc/opt/microsoft/docker-cimprov/td-agent-bit.conf;			    installer/conf/td-agent-bit.conf;                    644; root; root
+/etc/opt/microsoft/docker-cimprov/out_oms.conf;			            installer/conf/out_oms.conf;                    644; root; root
 
 %Links
 /opt/omi/lib/libcontainer.${{SHLIB_EXT}}; /opt/microsoft/docker-cimprov/lib/libcontainer.${{SHLIB_EXT}}; 644; root; root
@@ -79,6 +78,9 @@ MAINTAINER:              'Microsoft Corporation'
 /var/opt/microsoft/docker-cimprov/state/ImageInventory; 755; root; root
 /var/opt/microsoft/docker-cimprov/log;                  755; root; root
 
+/opt/td-agent-bit;                          755; root; root;sysdir
+/opt/td-agent-bit/bin;                      755; root; root;sysdir
+
 %Dependencies
 
 %Postinstall_10
@@ -88,15 +90,6 @@ WriteInstallInfo() {
 }
 WriteInstallInfo
 
-#Setup sudo permission for containerlogtailfilereader
-if [ -z $(cat /etc/sudoers.d/omsagent | grep /containerlogtailfilereader.rb) ]
-then
-    chmod +w /etc/sudoers.d/omsagent
-    echo "#run containerlogtailfilereader.rb for docker-provider" >> /etc/sudoers.d/omsagent
-    echo "omsagent ALL=(ALL) NOPASSWD: /opt/microsoft/omsagent/ruby/bin/ruby /opt/microsoft/omsagent/plugin/containerlogtailfilereader.rb *" >> /etc/sudoers.d/omsagent
-    chmod 440 /etc/sudoers.d/omsagent
-fi
-
 # Get the state file in place with proper permissions
 touch /var/opt/microsoft/docker-cimprov/state/LastEventQueryTime.txt
 chmod 644 /var/opt/microsoft/docker-cimprov/state/LastEventQueryTime.txt