Normalize CPU use graphs, improve load time

InstrumentsProcessor/Cookers/IdleTimeTracker.cs

@@ -0,0 +1,42 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using Microsoft.Performance.SDK;
+
+namespace InstrumentsProcessor.Cookers
+{
+    internal class IdleTimeTracker
+    {
+        private Timestamp[] CpuTimes = new Timestamp[10];
+
+        public TimestampDelta Advance(int coreId, Timestamp nextStart, TimestampDelta duration)
+        {
+            TimestampDelta gap = TimestampDelta.Zero;
+
+            if (coreId >= 0)
+            {
+                if (CpuTimes.Length <= coreId)
+                {
+                    Array.Resize(ref CpuTimes, coreId + 1);
+                }
+
+                Timestamp last = CpuTimes[coreId];
+                if (last != Timestamp.Zero)
+                {
+                    // ThreadState should not, but TimeProfile does have overlapping events
+                    // so we cannot in general make this assertion.
+                    // Debug.Assert(nextStart >= last, "Overlapping threads on same processor");
+                    if (nextStart > last)
+                    {
+                        gap = nextStart - last;
+                    }
+                }
+
+                CpuTimes[coreId] = nextStart + duration;
+            }
+
+            return gap;
+        }
+    }
+}

InstrumentsProcessor/Cookers/ThreadStateCooker.cs

@@ -1,24 +1,37 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-using Microsoft.Performance.SDK.Extensibility.DataCooking.SourceDataCooking;
-using Microsoft.Performance.SDK.Extensibility.DataCooking;
-using Microsoft.Performance.SDK.Extensibility;
 using Microsoft.Performance.SDK;
-using System.Collections.Generic;
-using System.Threading;
-using System;
+using Microsoft.Performance.SDK.Extensibility;
+using Microsoft.Performance.SDK.Extensibility.DataCooking;
+using Microsoft.Performance.SDK.Extensibility.DataCooking.SourceDataCooking;
 using InstrumentsProcessor.Parsing;
 using InstrumentsProcessor.Parsing.Events;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Threading;
 
 namespace InstrumentsProcessor.Cookers
 {
+    public sealed class ThreadStateAccumulator
+    {
+        public TimestampDelta Ready { get; set; }
+        public TimestampDelta Waiting { get; set; }
+    }
+
     public sealed class ThreadStateCooker
         : SourceDataCooker<Event, ParsingContext, Type>
     {
         public static readonly DataCookerPath DataCookerPath =
             DataCookerPath.ForSource(nameof(TraceSourceParser), nameof(ThreadStateCooker));
 
+        private readonly Dictionary<UInt64, ThreadStateAccumulator> WaitingThreads =
+            new Dictionary<UInt64, ThreadStateAccumulator>();
+
+        private IdleTimeTracker IdleTracker =
+            new IdleTimeTracker();
+
         public ThreadStateCooker()
             : base(DataCookerPath)
         {
@@ -38,9 +51,74 @@ public override DataProcessingResult CookDataElement(
             ParsingContext context,
             CancellationToken cancellationToken)
         {
-            ThreadStateEvents.Add((ThreadStateEvent)data);
+            var tsevent = (ThreadStateEvent)data;
+            if (tsevent.Thread == null)
+            {
+                return DataProcessingResult.Ignored;
+            }
+
+            UInt64 threadKey = tsevent.Thread.ThreadUniqueId();
+            WaitingThreads.TryGetValue(threadKey, out ThreadStateAccumulator tstate);
 
+            if (tsevent.State.Value == "Running")
+            {
+                if (tstate != null)
+                {
+                    // Merge ready and waiting times
+                    tsevent.Waiting = new Parsing.DataModels.TimestampDelta(tstate.Waiting);
+                    tsevent.Ready = new Parsing.DataModels.TimestampDelta(tstate.Ready);
+                    WaitingThreads.Remove(threadKey);
+                }
+
+                if (tsevent.Core != null)
+                {
+                    SynthesizeIdleTime(tsevent);
+                    ThreadStateEvents.Add(tsevent);
+                }
+            }
+            else if (tsevent.State.Value == "Terminated")
+            {
+                WaitingThreads.Remove(threadKey);
+            }
+            else
+            {
+                if (tstate == null)
+                {
+                    tstate = new ThreadStateAccumulator();
+                    WaitingThreads.Add(threadKey, tstate);
+                }
+
+                if (tsevent.State.Value == "Runnable")
+                {
+                    tstate.Ready += tsevent.Duration.Value;
+                }
+                else if (
+                    (tsevent.State.Value == "Blocked") ||
+                    (tsevent.State.Value == "Idle") ||
+                    (tsevent.State.Value == "Interrupted") ||
+                    (tsevent.State.Value == "Preempted"))
+                {
+                    tstate.Waiting += tsevent.Duration.Value;
+                }
+                else if (tsevent.State.Value != "Unknown")
+                {
+                    Debug.Assert(false, "Unknown thread state");
+                }
+            }
+
             return DataProcessingResult.Processed;
         }
+
+        private void SynthesizeIdleTime(ThreadStateEvent tsevent)
+        {
+            Timestamp curr = tsevent.Timestamp;
+            TimestampDelta idleGap = IdleTracker.Advance(tsevent.Core.CoreId, curr, tsevent.Duration.Value);
+
+            if (idleGap != TimestampDelta.Zero)
+            {
+                var idleEvent = ThreadStateEvent.MakeIdleEvent(tsevent.Core, curr - idleGap, idleGap);
+                ThreadStateEvents.Add(idleEvent);
+            }
+        }
     }
 }

InstrumentsProcessor/Cookers/TimeProfileCooker.cs

@@ -19,6 +19,9 @@ public sealed class TimeProfileCooker
         public static readonly DataCookerPath DataCookerPath =
             DataCookerPath.ForSource(nameof(TraceSourceParser), nameof(TimeProfileCooker));
 
+        private IdleTimeTracker IdleTracker =
+            new IdleTimeTracker();
+
         public TimeProfileCooker()
             : base(DataCookerPath)
         {
@@ -38,9 +41,28 @@ public override DataProcessingResult CookDataElement(
             ParsingContext context,
             CancellationToken cancellationToken)
         {
-            TimeProfileEvents.Add((TimeProfileEvent)data);
+            var tpevent = (TimeProfileEvent)data;
+            if ((tpevent.Thread == null) || (tpevent.Core == null))
+            {
+                return DataProcessingResult.Ignored;
+            }
+
+            SynthesizeIdleTime(tpevent);
+            TimeProfileEvents.Add(tpevent);
 
             return DataProcessingResult.Processed;
         }
+
+        private void SynthesizeIdleTime(TimeProfileEvent tpevent)
+        {
+            Timestamp curr = tpevent.StartTime;
+            TimestampDelta idleGap = IdleTracker.Advance(tpevent.Core.CoreId, curr, tpevent.Weight.Value);
+
+            if (idleGap != TimestampDelta.Zero)
+            {
+                var idleEvent = TimeProfileEvent.MakeIdleEvent(tpevent.Core, curr, idleGap);
+                TimeProfileEvents.Add(idleEvent);
+            }
+        }
     }
 }

InstrumentsProcessor/Parsing/DataModels/CPU.cs

@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+using System;
+using System.Reflection;
+using System.Xml;
+
+namespace InstrumentsProcessor.Parsing.DataModels
+{
+    public class CPU : IPropertyDeserializer
+    {
+        [CustomDeserialization]
+        public int CoreId { get; private set; }
+
+        [CustomDeserialization]
+        public string Core { get; private set; }
+
+        public object DeserializeProperty(XmlNode node, XmlParsingContext context, PropertyInfo property)
+        {
+            if (property.Name == "CoreId")
+            {
+                return int.Parse(node.InnerText);
+            }
+            else if (property.Name == "Core")
+            {
+                return node.Attributes["fmt"]?.Value;
+            }
+            else
+            {
+                throw new InvalidOperationException();
+            }
+        }
+    }
+}

InstrumentsProcessor/Parsing/DataModels/Integer.cs

@@ -9,6 +9,15 @@ namespace InstrumentsProcessor.Parsing.DataModels
 {
     public class Integer : IPropertyDeserializer
     {
+        public Integer()
+        {
+        }
+
+        public Integer(int value)
+        {
+            Value = value;
+        }
+
         [CustomDeserialization]
         public int Value { get; private set; }
 
@@ -22,6 +31,8 @@ public object DeserializeProperty(XmlNode node, XmlParsingContext context, Prope
             {
                 throw new InvalidOperationException();
             }
-        }
+        }
+
+        public static readonly XmlNodeDeserializer<Integer> Deserializer = new XmlNodeDeserializer<Integer>();
     }
 }

InstrumentsProcessor/Parsing/DataModels/Process.cs

@@ -4,20 +4,17 @@
 using System;
 using System.Reflection;
 using System.Xml;
-using System.Xml.Linq;
 
 namespace InstrumentsProcessor.Parsing.DataModels
 {
     public class Process : IPropertyDeserializer
     {
-        private static XmlNodeDeserializer<Integer> ProcessIdDeserializer = new XmlNodeDeserializer<Integer>();
         [CustomDeserialization]
         public Integer ProcessId { get; private set;}
 
         [CustomDeserialization]
         public string Name { get; private set; }
 
-        private static XmlNodeDeserializer<String> DeviceSessionDeserializer = new XmlNodeDeserializer<String>();
         [CustomDeserialization]
         public String DeviceSession { get; private set; }
 
@@ -27,24 +24,28 @@ public object DeserializeProperty(XmlNode node, XmlParsingContext context, Prope
             {
                 XmlNode propertyNode = node.ChildNodes.Count >= 1 ? node.ChildNodes[0] : null;
 
-                return ProcessIdDeserializer.Deserialize(propertyNode, context);
+                return Integer.Deserializer.Deserialize(propertyNode, context);
             }
             if (property.Name == "Name")
             {
-                XElement root = XElement.Parse(node.OuterXml);
-
-                return root.Attribute("fmt")?.Value;
+                return node.Attributes["fmt"]?.Value;
             }
             if (property.Name == "DeviceSession")
             {
                 XmlNode propertyNode = node.ChildNodes.Count >= 2 ? node.ChildNodes[1] : null;
 
-                return DeviceSessionDeserializer.Deserialize(propertyNode, context);
+                return String.Deserializer.Deserialize(propertyNode, context);
             }
             else
             {
                 throw new InvalidOperationException();
             }
         }
+
+        public static readonly Process IdleProcess = new Process
+        {
+            ProcessId = new Integer(-1),
+            Name = "Idle",
+        };
     }
 }

InstrumentsProcessor/Parsing/DataModels/String.cs

@@ -9,6 +9,15 @@ namespace InstrumentsProcessor.Parsing.DataModels
 {
     public class String : IPropertyDeserializer
     {
+        public String()
+        {
+        }
+
+        public String(string val)
+        {
+            Value = val;
+        }
+
         [CustomDeserialization]
         public string Value { get; private set; }
 
@@ -23,5 +32,7 @@ public object DeserializeProperty(XmlNode node, XmlParsingContext context, Prope
                 throw new InvalidOperationException();
             }
         }
+
+        public static readonly XmlNodeDeserializer<String> Deserializer = new XmlNodeDeserializer<String>();
     }
 }

InstrumentsProcessor/Parsing/DataModels/Thread.cs

@@ -1,11 +1,44 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+using System;
+using System.Reflection;
+using System.Xml;
+
 namespace InstrumentsProcessor.Parsing.DataModels
 {
-    public class Thread
+    public class Thread : IPropertyDeserializer
     {
         public Integer ThreadId { get; private set; }
         public Process Process { get; private set; }
+
+        [CustomDeserialization]
+        public string ThreadName { get; private set; }
+
+        public object DeserializeProperty(XmlNode node, XmlParsingContext context, PropertyInfo property)
+        {
+            if (property.Name == "ThreadName")
+            {
+                return node.Attributes["fmt"]?.Value ?? string.Empty;
+            }
+            else
+            {
+                throw new InvalidOperationException();
+            }
+        }
+
+        public System.UInt64 ThreadUniqueId()
+        {
+            System.UInt64 pid = (UInt32)Process.ProcessId.Value;
+            UInt32 tid = (UInt32)ThreadId.Value;
+            return (pid << 32) | tid;
+        }
+
+        public static readonly Thread IdleThread = new Thread
+        {
+            ThreadId = new Integer(-1),
+            Process = Process.IdleProcess,
+            ThreadName = "Idle thread"
+        };
     }
-}
+}