From 7025149503c0e0e95d0a0380568b34c05523b42b Mon Sep 17 00:00:00 2001 From: Max Charlamb Date: Wed, 6 Nov 2024 20:36:11 -0500 Subject: [PATCH 01/12] add ExecutionManager contract version --- .../ExecutionManagerBase.EEJitManager.cs} | 7 +- .../ExecutionManagerBase.cs} | 8 +- .../ExecutionManagerFactory.cs | 4 + .../ExecutionManager/ExecutionManager_1.cs | 16 + .../ExecutionManager/ExecutionManager_2.cs | 16 + .../ExecutionManager/Helpers/INibbleMap.cs | 15 + .../Helpers/NibbleMapHelpers.cs | 147 +++++++++ .../ExecutionManager/Helpers/NibbleMap_1.cs | 159 ++++++++++ .../ExecutionManager/Helpers/NibbleMap_2.cs | 130 ++++++++ .../Helpers}/RangeSectionMap.cs | 0 .../ExecutionManagerHelpers/NibbleMap.cs | 294 ------------------ .../ExecutionManagerTestBuilder.cs | 120 +------ .../ExecutionManagerTests.cs | 37 ++- .../NibbleMapTestBuilder.cs | 176 +++++++++++ .../NibbleMapTests.cs | 74 ++++- .../RangeSectionMapTests.cs | 2 +- 16 files changed, 779 insertions(+), 426 deletions(-) rename src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/{ExecutionManager_1.EEJitManager.cs => ExecutionManager/ExecutionManagerBase.EEJitManager.cs} (90%) rename src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/{ExecutionManager_1.cs => ExecutionManager/ExecutionManagerBase.cs} (96%) rename src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/{ => ExecutionManager}/ExecutionManagerFactory.cs (80%) create mode 100644 src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_1.cs create mode 100644 src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_2.cs create mode 100644 src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/INibbleMap.cs create mode 100644 src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapHelpers.cs create mode 100644 src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_1.cs create mode 100644 src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs rename src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/{ExecutionManagerHelpers => Contracts/ExecutionManager/Helpers}/RangeSectionMap.cs (100%) delete mode 100644 src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs rename src/native/managed/cdacreader/tests/{ => ExecutionManagerTests}/ExecutionManagerTestBuilder.cs (79%) rename src/native/managed/cdacreader/tests/{ => ExecutionManagerTests}/ExecutionManagerTests.cs (75%) create mode 100644 src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTestBuilder.cs rename src/native/managed/cdacreader/tests/{ => ExecutionManagerTests}/NibbleMapTests.cs (60%) rename src/native/managed/cdacreader/tests/{ => ExecutionManagerTests}/RangeSectionMapTests.cs (97%) diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager_1.EEJitManager.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerBase.EEJitManager.cs similarity index 90% rename from src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager_1.EEJitManager.cs rename to src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerBase.EEJitManager.cs index a3a828192e3fbd..eda3c4e3fb2476 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager_1.EEJitManager.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerBase.EEJitManager.cs @@ -4,15 +4,16 @@ using System; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; namespace Microsoft.Diagnostics.DataContractReader.Contracts; -internal readonly partial struct ExecutionManager_1 : IExecutionManager +internal partial class ExecutionManagerBase : IExecutionManager { private class EEJitManager : JitManager { - private readonly ExecutionManagerHelpers.NibbleMap _nibbleMap; - public EEJitManager(Target target, ExecutionManagerHelpers.NibbleMap nibbleMap) : base(target) + private readonly INibbleMap _nibbleMap; + public EEJitManager(Target target, INibbleMap nibbleMap) : base(target) { _nibbleMap = nibbleMap; } diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager_1.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerBase.cs similarity index 96% rename from src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager_1.cs rename to src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerBase.cs index f8a956f5af98cf..01d4925dccf26f 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager_1.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerBase.cs @@ -4,10 +4,12 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; namespace Microsoft.Diagnostics.DataContractReader.Contracts; -internal readonly partial struct ExecutionManager_1 : IExecutionManager +internal partial class ExecutionManagerBase : IExecutionManager + where T : INibbleMap { internal readonly Target _target; @@ -18,12 +20,12 @@ namespace Microsoft.Diagnostics.DataContractReader.Contracts; private readonly EEJitManager _eeJitManager; private readonly ReadyToRunJitManager _r2rJitManager; - public ExecutionManager_1(Target target, Data.RangeSectionMap topRangeSectionMap) + public ExecutionManagerBase(Target target, Data.RangeSectionMap topRangeSectionMap) { _target = target; _topRangeSectionMap = topRangeSectionMap; _rangeSectionMapLookup = ExecutionManagerHelpers.RangeSectionMap.Create(_target); - ExecutionManagerHelpers.NibbleMap nibbleMap = ExecutionManagerHelpers.NibbleMap.Create(_target); + INibbleMap nibbleMap = T.Create(_target); _eeJitManager = new EEJitManager(_target, nibbleMap); _r2rJitManager = new ReadyToRunJitManager(_target); } diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManagerFactory.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerFactory.cs similarity index 80% rename from src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManagerFactory.cs rename to src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerFactory.cs index 13f2ada40a499a..4f6aa4477f778c 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManagerFactory.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerFactory.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; namespace Microsoft.Diagnostics.DataContractReader.Contracts; @@ -14,6 +15,9 @@ IExecutionManager IContractFactory.CreateContract(Target targ return version switch { 1 => new ExecutionManager_1(target, rangeSectionMap), + + // The nibblemap algorithm was changed in version 2 + 2 => new ExecutionManager_2(target, rangeSectionMap), _ => default(ExecutionManager), }; } diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_1.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_1.cs new file mode 100644 index 00000000000000..8d46c68db75709 --- /dev/null +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_1.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; + +namespace Microsoft.Diagnostics.DataContractReader.Contracts; + +internal class ExecutionManager_1 : ExecutionManagerBase +{ + public ExecutionManager_1(Target target, Data.RangeSectionMap topRangeSectionMap) : base(target, topRangeSectionMap) + { + } +} diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_2.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_2.cs new file mode 100644 index 00000000000000..620114876a879a --- /dev/null +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_2.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; + +namespace Microsoft.Diagnostics.DataContractReader.Contracts; + +internal class ExecutionManager_2 : ExecutionManagerBase +{ + public ExecutionManager_2(Target target, Data.RangeSectionMap topRangeSectionMap) : base(target, topRangeSectionMap) + { + } +} diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/INibbleMap.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/INibbleMap.cs new file mode 100644 index 00000000000000..bc54bc7b006eca --- /dev/null +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/INibbleMap.cs @@ -0,0 +1,15 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Numerics; +using System.Diagnostics; +using System; + +namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; + +internal interface INibbleMap +{ + public static abstract INibbleMap Create(Target target); + + public TargetPointer FindMethodCode(Data.CodeHeapListNode heapListNode, TargetCodePointer jittedCodeAddress); +} diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapHelpers.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapHelpers.cs new file mode 100644 index 00000000000000..f195ff3bcd7753 --- /dev/null +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapHelpers.cs @@ -0,0 +1,147 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Numerics; +using System.Diagnostics; +using System; + +namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; + +internal static class NibbleMapHelpers +{ + // we will partition the address space into buckets of this many bytes. + // There is at most one code block header per bucket. + // Normally we would then need 5 bits (Log2(BytesPerBucket))to find the exact start address, + // but because code headers are aligned, we can store the offset in a 4-bit nibble instead and shift appropriately to compute + // the effective address + internal const ulong BytesPerBucket = 8 * MapUnit.SizeInBytes; + + // We load the map contents as 32-bit integers, which contains 8 4-bit nibbles. + // The algorithm will focus on each nibble in a map unit before moving on to the previous map unit + internal readonly struct MapUnit + { + public const int SizeInBytes = sizeof(uint); + public const ulong SizeInNibbles = 2 * SizeInBytes; + public readonly uint Value; + + public MapUnit(uint value) => Value = value; + + public override string ToString() => $"0x{Value:x}"; + + // Shift the next nibble into the least significant position. + public MapUnit ShiftNextNibble => new MapUnit(Value >>> 4); + + public const uint NibbleMask = 0x0Fu; + internal Nibble Nibble => new((int)(Value & NibbleMask)); + + public bool IsZero => Value == 0; + + // Assuming mapIdx is the index of a nibble within the current map unit, + // shift the unit so that nibble is in the least significant position and return the result. + public MapUnit FocusOnIndexedNibble(MapKey mapIdx) + { + int shift = mapIdx.GetNibbleShift(); + return new MapUnit (Value >>> shift); + } + } + + // Each nibble is a 4-bit integer that gives an offset within a bucket. + // We reserse 0 to mean that there is no method starting at any offset within a bucket + internal readonly struct Nibble + { + public readonly int Value; + + public Nibble(int value) + { + Debug.Assert (value >= 0 && value <= 0xF); + Value = value; + } + + public static Nibble Zero => new Nibble(0); + public bool IsEmpty => Value == 0; + + public ulong TargetByteOffset + { + get + { + Debug.Assert(Value != 0); + return (uint)(Value - 1) * MapUnit.SizeInBytes; + } + } + } + + // The key to the map is the index of an individual nibble + internal readonly struct MapKey + { + private readonly ulong MapIdx; + public MapKey(ulong mapIdx) => MapIdx = mapIdx; + public override string ToString() => $"0x{MapIdx:x}"; + + // The offset of the address in the target space that this map index represents + public ulong TargetByteOffset => MapIdx * BytesPerBucket; + + // The index of the map unit that contains this map index + public ulong ContainingMapUnitIndex => MapIdx / MapUnit.SizeInNibbles; + + // The offset of the map unit that contains this map index + public ulong ContainingMapUnitByteOffset => ContainingMapUnitIndex * MapUnit.SizeInBytes; + + // The map index is the index of a nibble within the map, this gives the index of that nibble within a map unit. + public int NibbleIndexInMapUnit => (int)(MapIdx & (MapUnit.SizeInNibbles - 1)); + + // go to the previous nibble + public MapKey Prev => new MapKey(MapIdx - 1); + + // to to the previous map unit + public MapKey PrevMapUnit => new MapKey(MapIdx - MapUnit.SizeInNibbles); + + // Get a MapKey that is aligned to the first nibble in the map unit that contains this map index + public MapKey AlignDownToMapUnit() =>new MapKey(MapIdx & (~(MapUnit.SizeInNibbles - 1))); + + // If the map index is less than the size of a map unit, we are in the first MapUnit and + // can stop searching + public bool InFirstMapUnit => MapIdx < MapUnit.SizeInNibbles; + + public bool IsZero => MapIdx == 0; + + // given the index of a nibble in the map, compute how much we have to shift a MapUnit to put that + // nibble in the least significant position. + internal int GetNibbleShift() + { + return 28 - (NibbleIndexInMapUnit * 4); // bit shift - 4 bits per nibble + } + + internal MapUnit ReadMapUnit(Target target, TargetPointer mapStart) + { + // Given a logical index into the map, compute the address in memory where that map unit is located + TargetPointer mapUnitAdderss = mapStart + ContainingMapUnitByteOffset; + return new MapUnit(target.Read(mapUnitAdderss)); + } + } + + // for tests + internal static int ComputeNibbleShift(MapKey mapIdx) => mapIdx.GetNibbleShift(); + + internal static TargetPointer RoundTripAddress(TargetPointer mapBase, TargetPointer currentPC) + { + TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value); + DecomposeAddress(relativeAddress, out MapKey mapIdx, out Nibble bucketByteIndex); + return GetAbsoluteAddress(mapBase, mapIdx, bucketByteIndex); + } + + // Given a base address, a map index, and a nibble value, compute the absolute address in memory + // that the index and nibble point to. + internal static TargetPointer GetAbsoluteAddress(TargetPointer baseAddress, MapKey mapIdx, Nibble nibble) + { + return baseAddress + mapIdx.TargetByteOffset + nibble.TargetByteOffset; + } + + // Given a relative address, decompose it into + // the bucket index and an offset within the bucket. + internal static void DecomposeAddress(TargetNUInt relative, out MapKey mapIdx, out Nibble bucketByteIndex) + { + mapIdx = new(relative.Value / BytesPerBucket); + int bucketByteOffset = (int)(relative.Value & (BytesPerBucket - 1)); + bucketByteIndex = new Nibble((bucketByteOffset / MapUnit.SizeInBytes) + 1); + } +} diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_1.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_1.cs new file mode 100644 index 00000000000000..a3a3a6f6f0b303 --- /dev/null +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_1.cs @@ -0,0 +1,159 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Numerics; +using System.Diagnostics; +using System; + +using static Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers.NibbleMapHelpers; + +namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; + +// Given a contiguous region of memory in which we lay out a collection of non-overlapping code blocks that are +// not too small (so that two adjacent ones aren't too close together) and where the start of each code block is aligned on some power of 2 and preceeded by a code header, +// we can break up the whole memory space into buckets of a fixed size (32-bytes in the current implementation), where +// each bucket either has a code block or not. +// Thinking of each code block address as a hex number, we can view it as: [index, offset] +// where each index gives us a bucket and the offset gives us the position of the header within the bucket. +// In the current implementation code must be 4 byte aligned therefore there are 8 possible offsets in a bucket. +// These are encoded as values 1-8 in the 4-bit nibble, with 0 reserved to mark the places in the map where a method doesn't start. +// +// To find the start of a method given an address we first convert it into a bucket index (giving the map unit) +// and an offset which we can then turn into the index of the nibble that covers that address. +// If the nibble is non-zero, we have the start of a method and it is near the given address. +// If the nibble is zero, we have to search backward first through the current map unit, and then through previous map +// units until we find a non-zero nibble. +// +// For example (all code addresses are relative to some unspecified base): +// Suppose there is code starting at address 304 (0x130) +// Then the map index will be 304 / 32 = 9 and the byte offset will be 304 % 32 = 16 +// Because addresses are 4-byte aligned, the nibble value will be 1 + 16 / 4 = 5 (we reserve 0 to mean no method). +// So the map unit containing index 9 will contain the value 0x5 << 24 (the map index 9 means we want the second nibble in the second map unit, and we number the nibbles starting from the most significant) +// Or 0x05000000 +// +// Now suppose we do a lookup for address 306 (0x132) +// The map index will be 306 / 32 = 9 and the byte offset will be 306 % 32 = 18 +// The nibble value will be 1 + 18 / 4 = 5 +// To do the lookup, we will load the map unit with index 9 (so the second 32-bit unit in the map) and get the value 0x05000000 +// We will then shift to focus on the nibble with map index 9 (which again has nibble shift 24), so +// the map unit will be 0x00000005 and we will get the nibble value 5. +// Therefore we know that there is a method start at map index 9, nibble value 5. +// The map index corresponds to an offset of 288 bytes and the nibble value 5 corresponds to an offset of (5 - 1) * 4 = 16 bytes +// So the method starts at offset 288 + 16 = 304, which is the address we were looking for. +// +// Now suppose we do a lookup for address 302 (0x12E) +// The map index will be 302 / 32 = 9 and the byte offset will be 302 % 32 = 14 +// The nibble value will be 1 + 14 / 4 = 4 +// To do the lookup, we will load the map unit containing map index 9 and get the value 0x05000000 +// We will then shift to focus on the nibble with map index 9 (which again has nibble shift 24), so we will get +// the nibble value 5. +// Therefore we know that there is a method start at map index 9, nibble value 5. +// But the address we're looking for is map index 9, nibble value 4. +// We know that methods can't start within 32-bytes of each other, so we know that the method we're looking for is not in the current nibble. +// We will then try to shift to the previous nibble in the map unit (0x00000005 >> 4 = 0x00000000) +// Therefore we know there is no method start at any map index in the current map unit. +// We will then align the map index to the start of the current map unit (map index 8) and move back to the previous map unit (map index 7) +// At that point, we scan backwards for non-zero map units. Since there are none, we return null. + +internal class NibbleMap_1 : INibbleMap +{ + private readonly Target _target; + + private NibbleMap_1(Target target) + { + _target = target; + } + + internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapStart, TargetCodePointer currentPC) + { + TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value); + DecomposeAddress(relativeAddress, out MapKey mapIdx, out Nibble bucketByteIndex); + + MapUnit t = mapIdx.ReadMapUnit(_target, mapStart); + + // shift the nibble we want to the least significant position + t = t.FocusOnIndexedNibble(mapIdx); + + // if the nibble is non-zero, we have found the start of a method, + // but we need to check that the start is before the current address, not after + if (!t.Nibble.IsEmpty && t.Nibble.Value <= bucketByteIndex.Value) + { + return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); + } + + // search backwards through the current map unit + // we processed the lsb nibble, move to the next one + t = t.ShiftNextNibble; + + // if there's any nibble set in the current unit, find it + if (!t.IsZero) + { + mapIdx = mapIdx.Prev; + while (t.Nibble.IsEmpty) + { + t = t.ShiftNextNibble; + mapIdx = mapIdx.Prev; + } + return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); + } + + // We finished the current map unit, we want to move to the previous one. + // But if we were in the first map unit, we can stop + if (mapIdx.InFirstMapUnit) + { + return TargetPointer.Null; + } + + // We're now done with the current map unit. + // Align the map index to the current map unit, then move back one nibble into the previous map unit + mapIdx = mapIdx.AlignDownToMapUnit(); + mapIdx = mapIdx.Prev; + + // read the map unit containing mapIdx and skip over it if it is all zeros + while (true) + { + t = mapIdx.ReadMapUnit(_target, mapStart); + if (!t.IsZero) + break; + if (mapIdx.InFirstMapUnit) + { + // we're at the first map unit and all the bits in the map unit are zero, + // there is no code header to find + return TargetPointer.Null; + } + mapIdx = mapIdx.PrevMapUnit; + } + + Debug.Assert(!t.IsZero); + + // move to the correct nibble in the map unit + while (!mapIdx.IsZero && t.Nibble.IsEmpty) + { + t = t.ShiftNextNibble; + mapIdx = mapIdx.Prev; + } + + if (mapIdx.IsZero && t.IsZero) + { + return TargetPointer.Null; + } + + return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); + } + + public static INibbleMap Create(Target target) + { + return new NibbleMap_1(target); + } + + public TargetPointer FindMethodCode(Data.CodeHeapListNode heapListNode, TargetCodePointer jittedCodeAddress) + { + if (jittedCodeAddress < heapListNode.StartAddress || jittedCodeAddress > heapListNode.EndAddress) + { + return TargetPointer.Null; + } + + return FindMethodCode(heapListNode.MapBase, heapListNode.HeaderMap, jittedCodeAddress); + } + +} diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs new file mode 100644 index 00000000000000..0cc0f69ecb8b39 --- /dev/null +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs @@ -0,0 +1,130 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Numerics; +using System.Diagnostics; +using System; + +using static Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers.NibbleMapHelpers; + +namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; + +// CoreCLR nibblemap with O(1) lookup time. + +internal class NibbleMap_2 : INibbleMap +{ + private readonly Target _target; + + private NibbleMap_2(Target target) + { + _target = target; + } + + internal static bool IsPointer(MapUnit mapUnit) + { + return (mapUnit.Value & MapUnit.NibbleMask) > 8; + } + + internal static TargetPointer DecodePointer(TargetPointer baseAddress, MapUnit mapUnit) + { + uint nibble = mapUnit.Value & MapUnit.NibbleMask; + uint relativePointer = (mapUnit.Value & ~MapUnit.NibbleMask) + ((nibble - 9) << 2); + return baseAddress + relativePointer; + } + + internal static uint EncodePointer(uint relativeAddress) + { + uint nibble = ((relativeAddress & MapUnit.NibbleMask) >>> 2) + 9; + return (relativeAddress & ~MapUnit.NibbleMask) + nibble; + } + + internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapStart, TargetCodePointer currentPC) + { + TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value); + DecomposeAddress(relativeAddress, out MapKey mapIdx, out Nibble bucketByteIndex); + + MapUnit t = mapIdx.ReadMapUnit(_target, mapStart); + + // if pointer, return value + if (IsPointer(t)) + { + return DecodePointer(mapBase, t); + } + + // shift the nibble we want to the least significant position + t = t.FocusOnIndexedNibble(mapIdx); + + // if the nibble is non-zero, we have found the start of a method, + // but we need to check that the start is before the current address, not after + if (!t.Nibble.IsEmpty && t.Nibble.Value <= bucketByteIndex.Value) + { + return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); + } + + // search backwards through the current map unit + // we processed the lsb nibble, move to the next one + t = t.ShiftNextNibble; + + // if there's any nibble set in the current unit, find it + if (!t.IsZero) + { + mapIdx = mapIdx.Prev; + while (t.Nibble.IsEmpty) + { + t = t.ShiftNextNibble; + mapIdx = mapIdx.Prev; + } + return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); + } + + // We finished the current map unit, we want to move to the previous one. + // But if we were in the first map unit, we can stop + if (mapIdx.InFirstMapUnit) + { + return TargetPointer.Null; + } + + // We're now done with the current map unit. + // Align the map index to the current map unit, then move back one nibble into the previous map unit + mapIdx = mapIdx.AlignDownToMapUnit(); + mapIdx = mapIdx.Prev; + + // read the map unit containing mapIdx and skip over it if it is all zeros + t = mapIdx.ReadMapUnit(_target, mapStart); + + // if t is not zero, we either have a pointer or a nibble + if (!t.IsZero) + { + if (IsPointer(t)) + { + return DecodePointer(mapBase, t); + } + + // move to the correct nibble in the map unit + while (!mapIdx.IsZero && t.Nibble.IsEmpty) + { + t = t.ShiftNextNibble; + mapIdx = mapIdx.Prev; + } + + return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); + } + + return TargetPointer.Null; + } + + public static INibbleMap Create(Target target) + { + return new NibbleMap_2(target); + } + + public TargetPointer FindMethodCode(Data.CodeHeapListNode heapListNode, TargetCodePointer jittedCodeAddress) + { + if (jittedCodeAddress < heapListNode.StartAddress || jittedCodeAddress > heapListNode.EndAddress) + { + return TargetPointer.Null; + } + + return FindMethodCode(heapListNode.MapBase, heapListNode.HeaderMap, jittedCodeAddress); + } +} diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/RangeSectionMap.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/RangeSectionMap.cs similarity index 100% rename from src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/RangeSectionMap.cs rename to src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/RangeSectionMap.cs diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs deleted file mode 100644 index aead8c6909394b..00000000000000 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs +++ /dev/null @@ -1,294 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Numerics; -using System.Diagnostics; -using System; - -namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; - -// Given a contiguous region of memory in which we lay out a collection of non-overlapping code blocks that are -// not too small (so that two adjacent ones aren't too close together) and where the start of each code block is preceeded by a code header aligned on some power of 2, -// we can break up the whole memory space into buckets of a fixed size (32-bytes in the current implementation), where -// each bucket either has a code block header or not. -// Thinking of each code block header address as a hex number, we can view it as: [index, offset, zeros] -// where each index gives us a bucket and the offset gives us the position of the header within the bucket. -// We encode each offset into a 4-bit nibble, reserving the special value 0 to mark the places in the map where a method doesn't start. -// -// To find the start of a method given an address we first convert it into a bucket index (giving the map unit) -// and an offset which we can then turn into the index of the nibble that covers that address. -// If the nibble is non-zero, we have the start of a method and it is near the given address. -// If the nibble is zero, we have to search backward first through the current map unit, and then through previous map -// units until we find a non-zero nibble. -// -// For example (all code addresses are relative to some unspecified base): -// Suppose there is code starting at address 304 (0x130) -// Then the map index will be 304 / 32 = 9 and the byte offset will be 304 % 32 = 16 -// Because addresses are 4-byte aligned, the nibble value will be 1 + 16 / 4 = 5 (we reserve 0 to mean no method). -// So the map unit containing index 9 will contain the value 0x5 << 24 (the map index 9 means we want the second nibble in the second map unit, and we number the nibbles starting from the most significant) -// Or 0x05000000 -// -// Now suppose we do a lookup for address 306 (0x132) -// The map index will be 306 / 32 = 9 and the byte offset will be 306 % 32 = 18 -// The nibble value will be 1 + 18 / 4 = 5 -// To do the lookup, we will load the map unit with index 9 (so the second 32-bit unit in the map) and get the value 0x05000000 -// We will then shift to focus on the nibble with map index 9 (which again has nibble shift 24), so -// the map unit will be 0x00000005 and we will get the nibble value 5. -// Therefore we know that there is a method start at map index 9, nibble value 5. -// The map index corresponds to an offset of 288 bytes and the nibble value 5 corresponds to an offset of (5 - 1) * 4 = 16 bytes -// So the method starts at offset 288 + 16 = 304, which is the address we were looking for. -// -// Now suppose we do a lookup for address 302 (0x12E) -// The map index will be 302 / 32 = 9 and the byte offset will be 302 % 32 = 14 -// The nibble value will be 1 + 14 / 4 = 4 -// To do the lookup, we will load the map unit containing map index 9 and get the value 0x05000000 -// We will then shift to focus on the nibble with map index 9 (which again has nibble shift 24), so we will get -// the nibble value 5. -// Therefore we know that there is a method start at map index 9, nibble value 5. -// But the address we're looking for is map index 9, nibble value 4. -// We know that methods can't start within 32-bytes of each other, so we know that the method we're looking for is not in the current nibble. -// We will then try to shift to the previous nibble in the map unit (0x00000005 >> 4 = 0x00000000) -// Therefore we know there is no method start at any map index in the current map unit. -// We will then align the map index to the start of the current map unit (map index 8) and move back to the previous map unit (map index 7) -// At that point, we scan backwards for non-zero map units. Since there are none, we return null. - -internal class NibbleMap -{ - // We load the map contents as 32-bit integers, which contains 8 4-bit nibbles. - // The algorithm will focus on each nibble in a map unit before moving on to the previous map unit - internal readonly struct MapUnit - { - public const int SizeInBytes = sizeof(uint); - public const ulong SizeInNibbles = 2 * SizeInBytes; - public readonly uint Value; - - public MapUnit(uint value) => Value = value; - - public override string ToString() => $"0x{Value:x}"; - - // Shift the next nibble into the least significant position. - public MapUnit ShiftNextNibble => new MapUnit(Value >>> 4); - - public const uint NibbleMask = 0x0Fu; - internal Nibble Nibble => new ((int)(Value & NibbleMask)); - - public bool IsZero => Value == 0; - - // Assuming mapIdx is the index of a nibble within the current map unit, - // shift the unit so that nibble is in the least significant position and return the result. - public MapUnit FocusOnIndexedNibble(MapKey mapIdx) - { - int shift = mapIdx.GetNibbleShift(); - return new MapUnit (Value >>> shift); - } - } - - // Each nibble is a 4-bit integer that gives an offset within a bucket. - // We reserse 0 to mean that there is no method starting at any offset within a bucket - internal readonly struct Nibble - { - public readonly int Value; - - public Nibble(int value) - { - Debug.Assert (value >= 0 && value <= 0xF); - Value = value; - } - - public static Nibble Zero => new Nibble(0); - public bool IsEmpty => Value == 0; - - public ulong TargetByteOffset - { - get - { - Debug.Assert(Value != 0); - return (uint)(Value - 1) * MapUnit.SizeInBytes; - } - } - } - - // The key to the map is the index of an individual nibble - internal readonly struct MapKey - { - private readonly ulong MapIdx; - public MapKey(ulong mapIdx) => MapIdx = mapIdx; - public override string ToString() => $"0x{MapIdx:x}"; - - // The offset of the address in the target space that this map index represents - public ulong TargetByteOffset => MapIdx * BytesPerBucket; - - // The index of the map unit that contains this map index - public ulong ContainingMapUnitIndex => MapIdx / MapUnit.SizeInNibbles; - - // The offset of the map unit that contains this map index - public ulong ContainingMapUnitByteOffset => ContainingMapUnitIndex * MapUnit.SizeInBytes; - - // The map index is the index of a nibble within the map, this gives the index of that nibble within a map unit. - public int NibbleIndexInMapUnit => (int)(MapIdx & (MapUnit.SizeInNibbles - 1)); - - // go to the previous nibble - public MapKey Prev => new MapKey(MapIdx - 1); - - // to to the previous map unit - public MapKey PrevMapUnit => new MapKey(MapIdx - MapUnit.SizeInNibbles); - - // Get a MapKey that is aligned to the first nibble in the map unit that contains this map index - public MapKey AlignDownToMapUnit() =>new MapKey(MapIdx & (~(MapUnit.SizeInNibbles - 1))); - - // If the map index is less than the size of a map unit, we are in the first MapUnit and - // can stop searching - public bool InFirstMapUnit => MapIdx < MapUnit.SizeInNibbles; - - public bool IsZero => MapIdx == 0; - - // given the index of a nibble in the map, compute how much we have to shift a MapUnit to put that - // nibble in the least significant position. - internal int GetNibbleShift() - { - return 28 - (NibbleIndexInMapUnit * 4); // bit shift - 4 bits per nibble - } - } - - public static NibbleMap Create(Target target) - { - return new NibbleMap(target); - } - - private readonly Target _target; - private NibbleMap(Target target) - { - _target = target; - } - - - - // we will partition the address space into buckets of this many bytes. - // There is at most one code block header per bucket. - // Normally we would then need 5 bits (Log2(BytesPerBucket))to find the exact start address, - // but because code headers are aligned, we can store the offset in a 4-bit nibble instead and shift appropriately to compute - // the effective address - private const ulong BytesPerBucket = 8 * MapUnit.SizeInBytes; - - - // for tests - internal static int ComputeNibbleShift(MapKey mapIdx) => mapIdx.GetNibbleShift(); - - // Given a base address, a map index, and a nibble value, compute the absolute address in memory - // that the index and nibble point to. - private static TargetPointer GetAbsoluteAddress(TargetPointer baseAddress, MapKey mapIdx, Nibble nibble) - { - return baseAddress + mapIdx.TargetByteOffset + nibble.TargetByteOffset; - } - - // Given a relative address, decompose it into - // the bucket index and an offset within the bucket. - private static void DecomposeAddress(TargetNUInt relative, out MapKey mapIdx, out Nibble bucketByteIndex) - { - mapIdx = new (relative.Value / BytesPerBucket); - int bucketByteOffset = (int)(relative.Value & (BytesPerBucket - 1)); - bucketByteIndex = new Nibble ((bucketByteOffset / MapUnit.SizeInBytes) + 1); - } - - internal static TargetPointer RoundTripAddress(TargetPointer mapBase, TargetPointer currentPC) - { - TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value); - DecomposeAddress(relativeAddress, out MapKey mapIdx, out Nibble bucketByteIndex); - return GetAbsoluteAddress(mapBase, mapIdx, bucketByteIndex); - } - - private MapUnit ReadMapUnit(TargetPointer mapStart, MapKey mapIdx) - { - // Given a logical index into the map, compute the address in memory where that map unit is located - TargetPointer mapUnitAdderss = mapStart + mapIdx.ContainingMapUnitByteOffset; - return new MapUnit (_target.Read(mapUnitAdderss)); - } - - internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapStart, TargetCodePointer currentPC) - { - TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value); - DecomposeAddress(relativeAddress, out MapKey mapIdx, out Nibble bucketByteIndex); - - MapUnit t = ReadMapUnit(mapStart, mapIdx); - - // shift the nibble we want to the least significant position - t = t.FocusOnIndexedNibble(mapIdx); - - // if the nibble is non-zero, we have found the start of a method, - // but we need to check that the start is before the current address, not after - if (!t.Nibble.IsEmpty && t.Nibble.Value <= bucketByteIndex.Value) - { - return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); - } - - // search backwards through the current map unit - // we processed the lsb nibble, move to the next one - t = t.ShiftNextNibble; - - // if there's any nibble set in the current unit, find it - if (!t.IsZero) - { - mapIdx = mapIdx.Prev; - while (t.Nibble.IsEmpty) - { - t = t.ShiftNextNibble; - mapIdx = mapIdx.Prev; - } - return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); - } - - // We finished the current map unit, we want to move to the previous one. - // But if we were in the first map unit, we can stop - if (mapIdx.InFirstMapUnit) - { - return TargetPointer.Null; - } - - // We're now done with the current map unit. - // Align the map index to the current map unit, then move back one nibble into the previous map unit - mapIdx = mapIdx.AlignDownToMapUnit(); - mapIdx = mapIdx.Prev; - - // read the map unit containing mapIdx and skip over it if it is all zeros - while (true) - { - t = ReadMapUnit(mapStart, mapIdx); - if (!t.IsZero) - break; - if (mapIdx.InFirstMapUnit) - { - // we're at the first map unit and all the bits in the map unit are zero, - // there is no code header to find - return TargetPointer.Null; - } - mapIdx = mapIdx.PrevMapUnit; - } - - Debug.Assert(!t.IsZero); - - // move to the correct nibble in the map unit - while (!mapIdx.IsZero && t.Nibble.IsEmpty) - { - t = t.ShiftNextNibble; - mapIdx = mapIdx.Prev; - } - - if (mapIdx.IsZero && t.IsZero) - { - return TargetPointer.Null; - } - - return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); - } - - public TargetPointer FindMethodCode(Data.CodeHeapListNode heapListNode, TargetCodePointer jittedCodeAddress) - { - if (jittedCodeAddress < heapListNode.StartAddress || jittedCodeAddress > heapListNode.EndAddress) - { - return TargetPointer.Null; - } - - return FindMethodCode(heapListNode.MapBase, heapListNode.HeaderMap, jittedCodeAddress); - } - -} diff --git a/src/native/managed/cdacreader/tests/ExecutionManagerTestBuilder.cs b/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTestBuilder.cs similarity index 79% rename from src/native/managed/cdacreader/tests/ExecutionManagerTestBuilder.cs rename to src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTestBuilder.cs index 614d63d3f97402..3f9d9e88590eca 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManagerTestBuilder.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTestBuilder.cs @@ -7,7 +7,7 @@ using InteriorMapValue = Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers.RangeSectionMap.InteriorMapValue; -namespace Microsoft.Diagnostics.DataContractReader.UnitTests; +namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManagerTests; internal class ExecutionManagerTestBuilder { @@ -36,103 +36,6 @@ public struct AllocationRange CodeHeaderStart = 0x0033_4000, CodeHeaderEnd = 0x0033_5000, }; - internal class NibbleMapTestBuilder - { - // This is the base address of the memory range that the map covers. - // The map works on code pointers as offsets from this address - // For testing we don't actually place anything into this space - private readonly TargetPointer MapBase; - - internal readonly MockTarget.Architecture Arch; - // this is the target memory representation of the nibble map itself - public readonly MockMemorySpace.HeapFragment NibbleMapFragment; - - public NibbleMapTestBuilder(TargetPointer mapBase, ulong mapRangeSize, TargetPointer mapStart,MockTarget.Architecture arch) - { - MapBase = mapBase; - Arch = arch; - int nibbleMapSize = (int)Addr2Pos(mapRangeSize); - NibbleMapFragment = new MockMemorySpace.HeapFragment { - Address = mapStart, - Data = new byte[nibbleMapSize], - Name = "Nibble Map", - }; - } - - public NibbleMapTestBuilder(TargetPointer mapBase, ulong mapRangeSize, MockMemorySpace.BumpAllocator allocator, MockTarget.Architecture arch) - { - MapBase = mapBase; - Arch = arch; - int nibbleMapSize = (int)Addr2Pos(mapRangeSize); - NibbleMapFragment = allocator.Allocate((ulong)nibbleMapSize, "Nibble Map"); - } - - const int Log2CodeAlign = 2; // N.B. this might be different on 64-bit in the future - const int Log2NibblesPerDword = 3; - const int Log2BytesPerBucket = Log2CodeAlign + Log2NibblesPerDword; - const int Log2NibbleSize = 2; - const int NibbleSize = 1 << Log2NibbleSize; - const uint NibblesPerDword = (8 * sizeof(uint)) >> Log2NibbleSize; - const uint NibblesPerDwordMask = NibblesPerDword - 1; - const uint BytesPerBucket = NibblesPerDword * (1 << Log2CodeAlign); - - const uint MaskBytesPerBucket = BytesPerBucket - 1; - - const uint NibbleMask = 0xf; - const int HighestNibbleBit = 32 - NibbleSize; - - const uint HighestNibbleMask = NibbleMask << HighestNibbleBit; - - private ulong Addr2Pos(ulong addr) - { - return addr >> Log2BytesPerBucket; - } - - private uint Addr2Offs(ulong addr) - { - return (uint) (((addr & MaskBytesPerBucket) >> Log2CodeAlign) + 1); - } - - private int Pos2ShiftCount (ulong addr) - { - return HighestNibbleBit - (int)((addr & NibblesPerDwordMask) << Log2NibbleSize); - } - public void AllocateCodeChunk(TargetCodePointer codeStart, int codeSize) - { - // paraphrased from EEJitManager::NibbleMapSetUnlocked - if (codeStart.Value < MapBase.Value) - { - throw new ArgumentException("Code start address is below the map base"); - } - ulong delta = codeStart.Value - MapBase.Value; - ulong pos = Addr2Pos(delta); - bool bSet = true; - uint value = bSet?Addr2Offs(delta):0; - - uint index = (uint) (pos >> Log2NibblesPerDword); - uint mask = ~(HighestNibbleMask >> (int)((pos & NibblesPerDwordMask) << Log2NibbleSize)); - - value = value << Pos2ShiftCount(pos); - - Span entry = NibbleMapFragment.Data.AsSpan((int)(index * sizeof(uint)), sizeof(uint)); - uint oldValue = TestPlaceholderTarget.ReadFromSpan(entry, Arch.IsLittleEndian); - - if (value != 0 && (oldValue & ~mask) != 0) - { - throw new InvalidOperationException("Overwriting existing offset"); - } - - uint newValue = (oldValue & mask) | value; - TestPlaceholderTarget.WriteToSpan(newValue, Arch.IsLittleEndian, entry); - } - } - - - internal static NibbleMapTestBuilder CreateNibbleMap(TargetPointer mapBase, ulong mapRangeSize, TargetPointer mapStart, MockTarget.Architecture arch) - { - return new NibbleMapTestBuilder(mapBase, mapRangeSize, mapStart, arch); - } - internal class RangeSectionMapTestBuilder { const ulong DefaultTopLevelAddress = 0x0000_1000u; // arbitrary @@ -149,7 +52,7 @@ internal class RangeSectionMapTestBuilder { } - public RangeSectionMapTestBuilder (TargetPointer topLevelAddress, MockMemorySpace.Builder builder) + public RangeSectionMapTestBuilder(TargetPointer topLevelAddress, MockMemorySpace.Builder builder) { _topLevelAddress = topLevelAddress; _builder = builder; @@ -271,6 +174,8 @@ public static RangeSectionMapTestBuilder CreateRangeSection(MockTarget.Architect return new RangeSectionMapTestBuilder(arch); } + internal readonly int _version; + internal MockMemorySpace.Builder Builder { get; } private readonly RangeSectionMapTestBuilder _rsmBuilder; @@ -280,12 +185,12 @@ public static RangeSectionMapTestBuilder CreateRangeSection(MockTarget.Architect internal readonly Dictionary TypeInfoCache = new(); - internal ExecutionManagerTestBuilder(MockTarget.Architecture arch, AllocationRange allocationRange) : this(new MockMemorySpace.Builder(new TargetTestHelpers(arch)), allocationRange) + internal ExecutionManagerTestBuilder(int version, MockTarget.Architecture arch, AllocationRange allocationRange) : this(version, new MockMemorySpace.Builder(new TargetTestHelpers(arch)), allocationRange) {} - - internal ExecutionManagerTestBuilder(MockMemorySpace.Builder builder, AllocationRange allocationRange, Dictionary? typeInfoCache = null) + internal ExecutionManagerTestBuilder(int version, MockMemorySpace.Builder builder, AllocationRange allocationRange, Dictionary? typeInfoCache = null) { + _version = version; Builder = builder; _rsmBuilder = new RangeSectionMapTestBuilder(ExecutionManagerCodeRangeMapAddress, builder); _rangeSectionMapAllocator = Builder.CreateAllocator(allocationRange.RangeSectionMapStart, allocationRange.RangeSectionMapEnd); @@ -353,10 +258,17 @@ internal static void AddToTypeInfoCache(TargetTestHelpers targetTestHelpers, Dic }; } - internal NibbleMapTestBuilder CreateNibbleMap(ulong codeRangeStart, uint codeRangeSize) + internal NibbleMapTestBuilderBase CreateNibbleMap(ulong codeRangeStart, uint codeRangeSize) { + NibbleMapTestBuilderBase nibBuilder = _version switch + { + 1 => new NibbleMapTestBuilder_1(codeRangeStart, codeRangeSize, _nibbleMapAllocator, Builder.TargetTestHelpers.Arch), + + // The nibblemap algorithm was changed in version 2 + 2 => new NibbleMapTestBuilder_2(codeRangeStart, codeRangeSize, _nibbleMapAllocator, Builder.TargetTestHelpers.Arch), + _ => throw new InvalidOperationException("Unknown version"), + }; - NibbleMapTestBuilder nibBuilder = new NibbleMapTestBuilder(codeRangeStart, codeRangeSize, _nibbleMapAllocator, Builder.TargetTestHelpers.Arch); Builder.AddHeapFragment(nibBuilder.NibbleMapFragment); return nibBuilder; } diff --git a/src/native/managed/cdacreader/tests/ExecutionManagerTests.cs b/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs similarity index 75% rename from src/native/managed/cdacreader/tests/ExecutionManagerTests.cs rename to src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs index 274addc986b861..39d861192a3d3f 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManagerTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs @@ -6,7 +6,8 @@ using Microsoft.Diagnostics.DataContractReader.Contracts; using System.Collections.Generic; using System; -namespace Microsoft.Diagnostics.DataContractReader.UnitTests; + +namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManagerTests; public class ExecutionManagerTests { @@ -15,7 +16,7 @@ internal class ExecutionManagerTestTarget : TestPlaceholderTarget { private readonly ulong _topRangeSectionMap; - public static ExecutionManagerTestTarget FromBuilder (ExecutionManagerTestBuilder emBuilder) + public static ExecutionManagerTestTarget FromBuilder(ExecutionManagerTestBuilder emBuilder) { var arch = emBuilder.Builder.TargetTestHelpers.Arch; ReadFromTargetDelegate reader = emBuilder.Builder.GetReadContext().ReadFromTarget; @@ -64,10 +65,10 @@ public override T ReadGlobal(string name) } [Theory] - [ClassData(typeof(MockTarget.StdArch))] - public void LookupNull(MockTarget.Architecture arch) + [MemberData(nameof(StdArchAllVersions))] + public void LookupNull(int version, MockTarget.Architecture arch) { - ExecutionManagerTestBuilder emBuilder = new (arch, ExecutionManagerTestBuilder.DefaultAllocationRange); + ExecutionManagerTestBuilder emBuilder = new (version, arch, ExecutionManagerTestBuilder.DefaultAllocationRange); emBuilder.MarkCreated(); var target = ExecutionManagerTestTarget.FromBuilder (emBuilder); @@ -78,10 +79,10 @@ public void LookupNull(MockTarget.Architecture arch) } [Theory] - [ClassData(typeof(MockTarget.StdArch))] - public void LookupNonNullMissing(MockTarget.Architecture arch) + [MemberData(nameof(StdArchAllVersions))] + public void LookupNonNullMissing(int version, MockTarget.Architecture arch) { - ExecutionManagerTestBuilder emBuilder = new (arch, ExecutionManagerTestBuilder.DefaultAllocationRange); + ExecutionManagerTestBuilder emBuilder = new (version, arch, ExecutionManagerTestBuilder.DefaultAllocationRange); emBuilder.MarkCreated(); var target = ExecutionManagerTestTarget.FromBuilder (emBuilder); @@ -92,8 +93,8 @@ public void LookupNonNullMissing(MockTarget.Architecture arch) } [Theory] - [ClassData(typeof(MockTarget.StdArch))] - public void LookupNonNullOneRangeOneMethod(MockTarget.Architecture arch) + [MemberData(nameof(StdArchAllVersions))] + public void LookupNonNullOneRangeOneMethod(int version, MockTarget.Architecture arch) { const ulong codeRangeStart = 0x0a0a_0000u; // arbitrary const uint codeRangeSize = 0xc000u; // arbitrary @@ -103,12 +104,12 @@ public void LookupNonNullOneRangeOneMethod(MockTarget.Architecture arch) TargetPointer expectedMethodDescAddress = new TargetPointer(0x0101_aaa0); - ExecutionManagerTestBuilder emBuilder = new (arch, ExecutionManagerTestBuilder.DefaultAllocationRange); + ExecutionManagerTestBuilder emBuilder = new(version, arch, ExecutionManagerTestBuilder.DefaultAllocationRange); var jittedCode = emBuilder.AllocateJittedCodeRange(codeRangeStart, codeRangeSize); TargetCodePointer methodStart = emBuilder.AddJittedMethod(jittedCode, methodSize, expectedMethodDescAddress); - ExecutionManagerTestBuilder.NibbleMapTestBuilder nibBuilder = emBuilder.CreateNibbleMap(codeRangeStart, codeRangeSize); + NibbleMapTestBuilderBase nibBuilder = emBuilder.CreateNibbleMap(codeRangeStart, codeRangeSize); nibBuilder.AllocateCodeChunk(methodStart, methodSize); TargetPointer codeHeapListNodeAddress = emBuilder.AddCodeHeapListNode(TargetPointer.Null, codeRangeStart, codeRangeStart + codeRangeSize, codeRangeStart, nibBuilder.NibbleMapFragment.Address); @@ -128,4 +129,16 @@ public void LookupNonNullOneRangeOneMethod(MockTarget.Architecture arch) TargetPointer actualMethodDesc = em.GetMethodDesc(eeInfo.Value); Assert.Equal(expectedMethodDescAddress, actualMethodDesc); } + + public static IEnumerable StdArchAllVersions() + { + const int highestVersion = 2; + foreach(object[] arr in new MockTarget.StdArch()) + { + MockTarget.Architecture arch = (MockTarget.Architecture)arr[0]; + for(int version = 1; version <= highestVersion; version++){ + yield return new object[] { version, arch }; + } + } + } } diff --git a/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTestBuilder.cs b/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTestBuilder.cs new file mode 100644 index 00000000000000..a0e9ac7f4d762a --- /dev/null +++ b/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTestBuilder.cs @@ -0,0 +1,176 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +using Microsoft.Diagnostics.DataContractReader.Contracts; +using System.Collections.Generic; +using System; +using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; + +namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManagerTests; + +internal abstract class NibbleMapTestBuilderBase +{ + // This is the base address of the memory range that the map covers. + // The map works on code pointers as offsets from this address + // For testing we don't actually place anything into this space + protected TargetPointer MapBase { get; init; } + + public MockTarget.Architecture Arch { get; init; } + + // this is the target memory representation of the nibble map itself + public MockMemorySpace.HeapFragment NibbleMapFragment { get; init; } + + protected const int Log2CodeAlign = 2; // N.B. this might be different on 64-bit in the future + protected const int Log2NibblesPerDword = 3; + protected const int Log2BytesPerBucket = Log2CodeAlign + Log2NibblesPerDword; + protected const int Log2NibbleSize = 2; + protected const int NibbleSize = 1 << Log2NibbleSize; + protected const uint NibblesPerDword = (8 * sizeof(uint)) >> Log2NibbleSize; + protected const uint NibblesPerDwordMask = NibblesPerDword - 1; + protected const uint BytesPerBucket = NibblesPerDword * (1 << Log2CodeAlign); + + protected const uint MaskBytesPerBucket = BytesPerBucket - 1; + + protected const uint NibbleMask = 0xf; + protected const int HighestNibbleBit = 32 - NibbleSize; + + protected const uint HighestNibbleMask = NibbleMask << HighestNibbleBit; + + protected ulong Addr2Pos(ulong addr) + { + return addr >> Log2BytesPerBucket; + } + + protected uint Addr2Offs(ulong addr) + { + return (uint) (((addr & MaskBytesPerBucket) >> Log2CodeAlign) + 1); + } + + protected int Pos2ShiftCount (ulong addr) + { + return HighestNibbleBit - (int)((addr & NibblesPerDwordMask) << Log2NibbleSize); + } + + public NibbleMapTestBuilderBase(TargetPointer mapBase, ulong mapRangeSize, TargetPointer mapStart, MockTarget.Architecture arch) + { + MapBase = mapBase; + Arch = arch; + int nibbleMapSize = (int)Addr2Pos(mapRangeSize); + NibbleMapFragment = new MockMemorySpace.HeapFragment { + Address = mapStart, + Data = new byte[nibbleMapSize], + Name = "Nibble Map", + }; + } + + public NibbleMapTestBuilderBase(TargetPointer mapBase, ulong mapRangeSize, MockMemorySpace.BumpAllocator allocator, MockTarget.Architecture arch) + { + MapBase = mapBase; + Arch = arch; + int nibbleMapSize = (int)Addr2Pos(mapRangeSize); + NibbleMapFragment = allocator.Allocate((ulong)nibbleMapSize, "Nibble Map"); + } + + public abstract void AllocateCodeChunk(TargetCodePointer codeStart, int codeSize); +} + +internal class NibbleMapTestBuilder_1 : NibbleMapTestBuilderBase +{ + public NibbleMapTestBuilder_1(TargetPointer mapBase, ulong mapRangeSize, TargetPointer mapStart, MockTarget.Architecture arch) + : base(mapBase, mapRangeSize, mapStart, arch) + { + } + + public NibbleMapTestBuilder_1(TargetPointer mapBase, ulong mapRangeSize, MockMemorySpace.BumpAllocator allocator, MockTarget.Architecture arch) + : base(mapBase, mapRangeSize, allocator, arch) + { + } + + public override void AllocateCodeChunk(TargetCodePointer codeStart, int codeSize) + { + // paraphrased from EEJitManager::NibbleMapSetUnlocked + if (codeStart.Value < MapBase.Value) + { + throw new ArgumentException("Code start address is below the map base"); + } + ulong delta = codeStart.Value - MapBase.Value; + ulong pos = Addr2Pos(delta); + bool bSet = true; + uint value = bSet?Addr2Offs(delta):0; + + uint index = (uint) (pos >> Log2NibblesPerDword); + uint mask = ~(HighestNibbleMask >> (int)((pos & NibblesPerDwordMask) << Log2NibbleSize)); + + value = value << Pos2ShiftCount(pos); + + Span entry = NibbleMapFragment.Data.AsSpan((int)(index * sizeof(uint)), sizeof(uint)); + uint oldValue = TestPlaceholderTarget.ReadFromSpan(entry, Arch.IsLittleEndian); + + if (value != 0 && (oldValue & ~mask) != 0) + { + throw new InvalidOperationException("Overwriting existing offset"); + } + + uint newValue = (oldValue & mask) | value; + TestPlaceholderTarget.WriteToSpan(newValue, Arch.IsLittleEndian, entry); + } +} + +internal class NibbleMapTestBuilder_2 : NibbleMapTestBuilderBase +{ + public NibbleMapTestBuilder_2(TargetPointer mapBase, ulong mapRangeSize, TargetPointer mapStart, MockTarget.Architecture arch) + : base(mapBase, mapRangeSize, mapStart, arch) + { + } + + public NibbleMapTestBuilder_2(TargetPointer mapBase, ulong mapRangeSize, MockMemorySpace.BumpAllocator allocator, MockTarget.Architecture arch) + : base(mapBase, mapRangeSize, allocator, arch) + { + } + + public override void AllocateCodeChunk(TargetCodePointer codeStart, int codeSize) + { + // paraphrased from EEJitManager::NibbleMapSetUnlocked + if (codeStart.Value < MapBase.Value) + { + throw new ArgumentException("Code start address is below the map base"); + } + ulong delta = codeStart.Value - MapBase.Value; + + ulong pos = Addr2Pos(delta); + uint value = Addr2Offs(delta); + + uint index = (uint) (pos >> Log2NibblesPerDword); + uint mask = ~(HighestNibbleMask >> (int)((pos & NibblesPerDwordMask) << Log2NibbleSize)); + + value = value << Pos2ShiftCount(pos); + + Span entry = NibbleMapFragment.Data.AsSpan((int)(index * sizeof(uint)), sizeof(uint)); + uint oldValue = TestPlaceholderTarget.ReadFromSpan(entry, Arch.IsLittleEndian); + + if (value != 0 && (oldValue & ~mask) != 0) + { + throw new InvalidOperationException("Overwriting existing offset"); + } + + uint newValue = (oldValue & mask) | value; + TestPlaceholderTarget.WriteToSpan(newValue, Arch.IsLittleEndian, entry); + + ulong firstByteAfterMethod = delta + (uint)codeSize; + uint encodedPointer = NibbleMap_2.EncodePointer((uint)delta); + index++; + while((index + 1) * 256 <= firstByteAfterMethod) + { + entry = NibbleMapFragment.Data.AsSpan((int)(index * sizeof(uint)), sizeof(uint)); + oldValue = TestPlaceholderTarget.ReadFromSpan(entry, Arch.IsLittleEndian); + if(oldValue != 0) + { + throw new InvalidOperationException("Overwriting existing offset"); + } + TestPlaceholderTarget.WriteToSpan(encodedPointer, Arch.IsLittleEndian, entry); + index++; + } + } +} diff --git a/src/native/managed/cdacreader/tests/NibbleMapTests.cs b/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTests.cs similarity index 60% rename from src/native/managed/cdacreader/tests/NibbleMapTests.cs rename to src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTests.cs index e887ee4e2cdb14..5b8a4e8cb317fe 100644 --- a/src/native/managed/cdacreader/tests/NibbleMapTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTests.cs @@ -7,9 +7,9 @@ using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; using System.Diagnostics; -namespace Microsoft.Diagnostics.DataContractReader.UnitTests; +namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManagerTests; -public class NibbleMapTests +public class NibbleMapTestsBase { internal class NibbleMapTestTarget : TestPlaceholderTarget { @@ -22,14 +22,16 @@ public NibbleMapTestTarget(MockTarget.Architecture arch, MockMemorySpace.ReadCon } - internal static NibbleMapTestTarget CreateTarget(ExecutionManagerTestBuilder.NibbleMapTestBuilder nibbleMapTestBuilder) + internal static NibbleMapTestTarget CreateTarget(NibbleMapTestBuilderBase nibbleMapTestBuilder) { return new NibbleMapTestTarget(nibbleMapTestBuilder.Arch, new MockMemorySpace.ReadContext() { HeapFragments = new[] { nibbleMapTestBuilder.NibbleMapFragment } }); } +} - +public class NibbleMapTests_1 : NibbleMapTestsBase +{ [Fact] public void RoundTripAddressTest() { @@ -37,7 +39,7 @@ public void RoundTripAddressTest() uint delta = 0x10u; for (TargetPointer p = mapBase; p < mapBase + 0x1000; p += delta) { - TargetPointer actual = NibbleMap.RoundTripAddress(mapBase, p); + TargetPointer actual = NibbleMapHelpers.RoundTripAddress(mapBase, p); Assert.Equal(p, actual); } } @@ -57,8 +59,8 @@ public void ExhaustiveNibbbleShifts(ulong irrelevant) int expectedShift = 28; for (int i = 0; i < 255; i++) { - NibbleMap.MapKey input = new (irrelevant + (ulong)i); - int actualShift = NibbleMap.ComputeNibbleShift(input); + NibbleMapHelpers.MapKey input = new (irrelevant + (ulong)i); + int actualShift = NibbleMapHelpers.ComputeNibbleShift(input); Assert.True(expectedShift == actualShift, $"Expected {expectedShift}, got {actualShift} for input {input}"); expectedShift -= 4; if (expectedShift == -4) @@ -81,7 +83,7 @@ public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) /// this is how big the address space is that the map covers const uint MapRangeSize = 0x1000; TargetPointer MapEnd = mapBase + MapRangeSize; - var builder = ExecutionManagerTestBuilder.CreateNibbleMap(mapBase, MapRangeSize, mapStart, arch); + var builder = new NibbleMapTestBuilder_1(mapBase, MapRangeSize, mapStart, arch); // don't put the code too close to the start - the NibbleMap bails if the code is too close to the start of the range TargetCodePointer inputPC = new(mapBase + 0x0200u); @@ -91,7 +93,7 @@ public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) // TESTCASE: - NibbleMap map = NibbleMap.Create(target); + NibbleMap_1 map = (NibbleMap_1)NibbleMap_1.Create(target); Assert.NotNull(map); TargetPointer methodCode = map.FindMethodCode(mapBase, mapStart, inputPC); @@ -124,6 +126,60 @@ public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) } } +} + +public class NibbleMapTests_2 : NibbleMapTestsBase +{ + [Theory] + [ClassData(typeof(MockTarget.StdArch))] + public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) + { + // SETUP: + + // this is the beginning of the address range where code pointers might point + TargetPointer mapBase = new(0x5f5f_0000u); + // this is the beginning of the nibble map itself + TargetPointer mapStart = new(0x0456_1000u); + /// this is how big the address space is that the map covers + const uint MapRangeSize = 0x1000; + TargetPointer MapEnd = mapBase + MapRangeSize; + var builder = new NibbleMapTestBuilder_2(mapBase, MapRangeSize, mapStart, arch); + // don't put the code too close to the start - the NibbleMap bails if the code is too close to the start of the range + TargetCodePointer inputPC = new(mapBase + 0x0200u); + int codeSize = 0x400; + builder.AllocateCodeChunk (inputPC, codeSize); + NibbleMapTestTarget target = CreateTarget(builder); + // TESTCASE: + + NibbleMap_2 map = (NibbleMap_2)NibbleMap_2.Create(target); + Assert.NotNull(map); + + TargetPointer methodCode = map.FindMethodCode(mapBase, mapStart, inputPC); + Assert.Equal(inputPC.Value, methodCode.Value); + + // All addresses in the code chunk should map to the same method + for (int i = 0; i < codeSize; i++) + { + methodCode = map.FindMethodCode(mapBase, mapStart, inputPC.Value + (uint)i); + // we should always find the beginning of the method + Assert.Equal(inputPC.Value, methodCode.Value); + } + + // All addresses before the code chunk should return null + for (ulong i = mapBase; i < inputPC; i++) + { + methodCode = map.FindMethodCode(mapBase, mapStart, i); + Assert.Equal(0u, methodCode.Value); + } + + // All addresses more than 512 bytes after the code chunk should return null + for (ulong i = inputPC.Value + (uint)codeSize + 512; i < MapEnd; i++) + { + methodCode = map.FindMethodCode(mapBase, mapStart, i); + Assert.Equal(0u, methodCode.Value); + } + } } + diff --git a/src/native/managed/cdacreader/tests/RangeSectionMapTests.cs b/src/native/managed/cdacreader/tests/ExecutionManagerTests/RangeSectionMapTests.cs similarity index 97% rename from src/native/managed/cdacreader/tests/RangeSectionMapTests.cs rename to src/native/managed/cdacreader/tests/ExecutionManagerTests/RangeSectionMapTests.cs index e13e670189ba62..32cb330fdd0478 100644 --- a/src/native/managed/cdacreader/tests/RangeSectionMapTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManagerTests/RangeSectionMapTests.cs @@ -7,7 +7,7 @@ using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; -namespace Microsoft.Diagnostics.DataContractReader.UnitTests; +namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManagerTests; public class RangeSectionMapTests { From adfbcb31441c53ae2fa0e07990a3a44bb45b118a Mon Sep 17 00:00:00 2001 From: Max Charlamb Date: Thu, 7 Nov 2024 10:23:51 -0500 Subject: [PATCH 02/12] add docs --- .../ExecutionManager/Helpers/NibbleMap_2.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs index 0cc0f69ecb8b39..d71c3a42404a5f 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs @@ -10,6 +10,24 @@ namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; // CoreCLR nibblemap with O(1) lookup time. +// +// Implementation very similar to NibbleMap_1, but with the addition of writing relative pointers +// into the nibblemap whenever a code block completely covers a DWORD. This allows for O(1) lookup +// with the cost of O(n) write time. +// +// Pointers are encoded using the top 28 bits of the DWORD normally, the bottom 4 bits of the pointer +// are reduced to 2 bits due to 4 byte code offset and encoded in bits 28 .. 31 of the DWORD with values +// 9-12. This is used to differentiate nibble values and pointer DWORDs. +// +// To read the nibblemap, we check if the DWORD is a pointer. If so, then we know the value currentPC is +// part of a managed code block beginning at the mapBase + decoded pointer. If the DWORD is empty +// (no pointer or previous nibbles), then we only need to read the previous DWORD. If that DWORD is empty, +// then we must not be in a managed function. Otherwise the write algorithm would have written a relative +// pointer in the DWORD. +// +// Note, a currentPC pointing to bytes outside a function have undefined lookup behavior. +// In this implementation we may "extend" the lookup period of a function several hundred bytes +// if there is not another function following it. internal class NibbleMap_2 : INibbleMap { From d6d0d71968fad8d35200ca78c458896ee9008fc4 Mon Sep 17 00:00:00 2001 From: Max Charlamb Date: Thu, 7 Nov 2024 13:21:43 -0500 Subject: [PATCH 03/12] improve tests --- .../ExecutionManagerTests.cs | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs b/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs index 39d861192a3d3f..2e70bbedae5f11 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs @@ -22,10 +22,10 @@ public static ExecutionManagerTestTarget FromBuilder(ExecutionManagerTestBuilder ReadFromTargetDelegate reader = emBuilder.Builder.GetReadContext().ReadFromTarget; var topRangeSectionMap = ExecutionManagerTestBuilder.ExecutionManagerCodeRangeMapAddress; var typeInfo = emBuilder.TypeInfoCache; - return new ExecutionManagerTestTarget(arch, reader, topRangeSectionMap, typeInfo); + return new ExecutionManagerTestTarget(emBuilder._version, arch, reader, topRangeSectionMap, typeInfo); } - public ExecutionManagerTestTarget(MockTarget.Architecture arch, ReadFromTargetDelegate dataReader, TargetPointer topRangeSectionMap, Dictionary typeInfoCache) : base(arch) + public ExecutionManagerTestTarget(int version, MockTarget.Architecture arch, ReadFromTargetDelegate dataReader, TargetPointer topRangeSectionMap, Dictionary typeInfoCache) : base(arch) { _topRangeSectionMap = topRangeSectionMap; SetDataReader(dataReader); @@ -33,7 +33,7 @@ public ExecutionManagerTestTarget(MockTarget.Architecture arch, ReadFromTargetDe SetDataCache(new DefaultDataCache(this)); IContractFactory emfactory = new ExecutionManagerFactory(); SetContracts(new TestRegistry() { - ExecutionManagerContract = new (() => emfactory.CreateContract(this, 1)), + ExecutionManagerContract = new (() => emfactory.CreateContract(this, version)), }); } public override TargetPointer ReadGlobalPointer(string global) @@ -98,7 +98,7 @@ public void LookupNonNullOneRangeOneMethod(int version, MockTarget.Architecture { const ulong codeRangeStart = 0x0a0a_0000u; // arbitrary const uint codeRangeSize = 0xc000u; // arbitrary - int methodSize = 0x100; // arbitrary + int methodSize = 0x450; // arbitrary TargetPointer jitManagerAddress = new (0x000b_ff00); // arbitrary @@ -120,7 +120,7 @@ public void LookupNonNullOneRangeOneMethod(int version, MockTarget.Architecture var target = ExecutionManagerTestTarget.FromBuilder(emBuilder); - // test + // test at method start var em = target.Contracts.ExecutionManager; Assert.NotNull(em); @@ -128,6 +128,16 @@ public void LookupNonNullOneRangeOneMethod(int version, MockTarget.Architecture Assert.NotNull(eeInfo); TargetPointer actualMethodDesc = em.GetMethodDesc(eeInfo.Value); Assert.Equal(expectedMethodDescAddress, actualMethodDesc); + + // test middle of method + eeInfo = em.GetCodeBlockHandle(methodStart + 0x250); + Assert.NotNull(eeInfo); + Assert.Equal(expectedMethodDescAddress, actualMethodDesc); + + // test end of method + eeInfo = em.GetCodeBlockHandle(methodStart + 0x450 - 1); + Assert.NotNull(eeInfo); + Assert.Equal(expectedMethodDescAddress, actualMethodDesc); } public static IEnumerable StdArchAllVersions() From 3994c77fc37f1dfba4642576bcf6de888c545abf Mon Sep 17 00:00:00 2001 From: Max Charlamb Date: Thu, 7 Nov 2024 14:12:09 -0500 Subject: [PATCH 04/12] _version -> Version --- .../tests/ExecutionManagerTests/ExecutionManagerTestBuilder.cs | 2 +- .../tests/ExecutionManagerTests/ExecutionManagerTests.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTestBuilder.cs b/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTestBuilder.cs index 3f9d9e88590eca..d27c1abce0e6cd 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTestBuilder.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTestBuilder.cs @@ -174,7 +174,7 @@ public static RangeSectionMapTestBuilder CreateRangeSection(MockTarget.Architect return new RangeSectionMapTestBuilder(arch); } - internal readonly int _version; + internal int Verion { get;} internal MockMemorySpace.Builder Builder { get; } private readonly RangeSectionMapTestBuilder _rsmBuilder; diff --git a/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs b/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs index 2e70bbedae5f11..ef48a476ed083a 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs @@ -22,7 +22,7 @@ public static ExecutionManagerTestTarget FromBuilder(ExecutionManagerTestBuilder ReadFromTargetDelegate reader = emBuilder.Builder.GetReadContext().ReadFromTarget; var topRangeSectionMap = ExecutionManagerTestBuilder.ExecutionManagerCodeRangeMapAddress; var typeInfo = emBuilder.TypeInfoCache; - return new ExecutionManagerTestTarget(emBuilder._version, arch, reader, topRangeSectionMap, typeInfo); + return new ExecutionManagerTestTarget(emBuilder.Verion, arch, reader, topRangeSectionMap, typeInfo); } public ExecutionManagerTestTarget(int version, MockTarget.Architecture arch, ReadFromTargetDelegate dataReader, TargetPointer topRangeSectionMap, Dictionary typeInfoCache) : base(arch) From f3b393d4698e2470d3b1e697026133beec67ba87 Mon Sep 17 00:00:00 2001 From: Max Charlamb Date: Thu, 7 Nov 2024 14:12:26 -0500 Subject: [PATCH 05/12] comments --- .../Helpers/NibbleMapHelpers.cs | 50 +++++++++++-------- .../ExecutionManager/Helpers/NibbleMap_1.cs | 8 +-- .../ExecutionManager/Helpers/NibbleMap_2.cs | 4 +- .../NibbleMapTestBuilder.cs | 16 +++--- 4 files changed, 43 insertions(+), 35 deletions(-) diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapHelpers.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapHelpers.cs index f195ff3bcd7753..5d5f9870d88796 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapHelpers.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapHelpers.cs @@ -32,16 +32,16 @@ internal readonly struct MapUnit public MapUnit ShiftNextNibble => new MapUnit(Value >>> 4); public const uint NibbleMask = 0x0Fu; - internal Nibble Nibble => new((int)(Value & NibbleMask)); + internal Nibble Nibble => new(Value & NibbleMask); - public bool IsZero => Value == 0; + public bool IsEmpty => Value == 0; // Assuming mapIdx is the index of a nibble within the current map unit, // shift the unit so that nibble is in the least significant position and return the result. public MapUnit FocusOnIndexedNibble(MapKey mapIdx) { - int shift = mapIdx.GetNibbleShift(); - return new MapUnit (Value >>> shift); + uint shift = mapIdx.GetNibbleShift(); + return new MapUnit(Value >>> (int)shift); } } @@ -49,11 +49,11 @@ public MapUnit FocusOnIndexedNibble(MapKey mapIdx) // We reserse 0 to mean that there is no method starting at any offset within a bucket internal readonly struct Nibble { - public readonly int Value; + public readonly uint Value; - public Nibble(int value) + public Nibble(uint value) { - Debug.Assert (value >= 0 && value <= 0xF); + Debug.Assert(value <= 0xF); Value = value; } @@ -73,40 +73,48 @@ public ulong TargetByteOffset // The key to the map is the index of an individual nibble internal readonly struct MapKey { - private readonly ulong MapIdx; - public MapKey(ulong mapIdx) => MapIdx = mapIdx; - public override string ToString() => $"0x{MapIdx:x}"; + private readonly ulong _mapIdx; + public MapKey(ulong mapIdx) => _mapIdx = mapIdx; + public override string ToString() => $"0x{_mapIdx:x}"; // The offset of the address in the target space that this map index represents - public ulong TargetByteOffset => MapIdx * BytesPerBucket; + public ulong TargetByteOffset => _mapIdx * BytesPerBucket; // The index of the map unit that contains this map index - public ulong ContainingMapUnitIndex => MapIdx / MapUnit.SizeInNibbles; + public ulong ContainingMapUnitIndex => _mapIdx / MapUnit.SizeInNibbles; // The offset of the map unit that contains this map index public ulong ContainingMapUnitByteOffset => ContainingMapUnitIndex * MapUnit.SizeInBytes; // The map index is the index of a nibble within the map, this gives the index of that nibble within a map unit. - public int NibbleIndexInMapUnit => (int)(MapIdx & (MapUnit.SizeInNibbles - 1)); + public uint NibbleIndexInMapUnit => (uint)(_mapIdx & (MapUnit.SizeInNibbles - 1)); // go to the previous nibble - public MapKey Prev => new MapKey(MapIdx - 1); + public MapKey Prev + { + get + { + Debug.Assert(_mapIdx > 0); + return new MapKey(_mapIdx - 1); + } + + } // to to the previous map unit - public MapKey PrevMapUnit => new MapKey(MapIdx - MapUnit.SizeInNibbles); + public MapKey PrevMapUnit => new MapKey(_mapIdx - MapUnit.SizeInNibbles); // Get a MapKey that is aligned to the first nibble in the map unit that contains this map index - public MapKey AlignDownToMapUnit() =>new MapKey(MapIdx & (~(MapUnit.SizeInNibbles - 1))); + public MapKey AlignDownToMapUnit() =>new MapKey(_mapIdx & (~(MapUnit.SizeInNibbles - 1))); // If the map index is less than the size of a map unit, we are in the first MapUnit and // can stop searching - public bool InFirstMapUnit => MapIdx < MapUnit.SizeInNibbles; + public bool InFirstMapUnit => _mapIdx < MapUnit.SizeInNibbles; - public bool IsZero => MapIdx == 0; + public bool IsZero => _mapIdx == 0; // given the index of a nibble in the map, compute how much we have to shift a MapUnit to put that // nibble in the least significant position. - internal int GetNibbleShift() + internal uint GetNibbleShift() { return 28 - (NibbleIndexInMapUnit * 4); // bit shift - 4 bits per nibble } @@ -120,7 +128,7 @@ internal MapUnit ReadMapUnit(Target target, TargetPointer mapStart) } // for tests - internal static int ComputeNibbleShift(MapKey mapIdx) => mapIdx.GetNibbleShift(); + internal static uint ComputeNibbleShift(MapKey mapIdx) => mapIdx.GetNibbleShift(); internal static TargetPointer RoundTripAddress(TargetPointer mapBase, TargetPointer currentPC) { @@ -141,7 +149,7 @@ internal static TargetPointer GetAbsoluteAddress(TargetPointer baseAddress, MapK internal static void DecomposeAddress(TargetNUInt relative, out MapKey mapIdx, out Nibble bucketByteIndex) { mapIdx = new(relative.Value / BytesPerBucket); - int bucketByteOffset = (int)(relative.Value & (BytesPerBucket - 1)); + uint bucketByteOffset = (uint)(relative.Value & (BytesPerBucket - 1)); bucketByteIndex = new Nibble((bucketByteOffset / MapUnit.SizeInBytes) + 1); } } diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_1.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_1.cs index a3a3a6f6f0b303..6550fa72c220ff 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_1.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_1.cs @@ -86,7 +86,7 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt t = t.ShiftNextNibble; // if there's any nibble set in the current unit, find it - if (!t.IsZero) + if (!t.IsEmpty) { mapIdx = mapIdx.Prev; while (t.Nibble.IsEmpty) @@ -113,7 +113,7 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt while (true) { t = mapIdx.ReadMapUnit(_target, mapStart); - if (!t.IsZero) + if (!t.IsEmpty) break; if (mapIdx.InFirstMapUnit) { @@ -124,7 +124,7 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt mapIdx = mapIdx.PrevMapUnit; } - Debug.Assert(!t.IsZero); + Debug.Assert(!t.IsEmpty); // move to the correct nibble in the map unit while (!mapIdx.IsZero && t.Nibble.IsEmpty) @@ -133,7 +133,7 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt mapIdx = mapIdx.Prev; } - if (mapIdx.IsZero && t.IsZero) + if (mapIdx.IsZero && t.IsEmpty) { return TargetPointer.Null; } diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs index d71c3a42404a5f..104480c1e28d87 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs @@ -84,7 +84,7 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt t = t.ShiftNextNibble; // if there's any nibble set in the current unit, find it - if (!t.IsZero) + if (!t.IsEmpty) { mapIdx = mapIdx.Prev; while (t.Nibble.IsEmpty) @@ -111,7 +111,7 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt t = mapIdx.ReadMapUnit(_target, mapStart); // if t is not zero, we either have a pointer or a nibble - if (!t.IsZero) + if (!t.IsEmpty) { if (IsPointer(t)) { diff --git a/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTestBuilder.cs b/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTestBuilder.cs index a0e9ac7f4d762a..6238ccb484e915 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTestBuilder.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTestBuilder.cs @@ -22,12 +22,12 @@ internal abstract class NibbleMapTestBuilderBase // this is the target memory representation of the nibble map itself public MockMemorySpace.HeapFragment NibbleMapFragment { get; init; } - protected const int Log2CodeAlign = 2; // N.B. this might be different on 64-bit in the future + protected const int Log2CodeAlign = 2; // This might be different on 64-bit in the future protected const int Log2NibblesPerDword = 3; protected const int Log2BytesPerBucket = Log2CodeAlign + Log2NibblesPerDword; protected const int Log2NibbleSize = 2; protected const int NibbleSize = 1 << Log2NibbleSize; - protected const uint NibblesPerDword = (8 * sizeof(uint)) >> Log2NibbleSize; + protected const uint NibblesPerDword = (8 * sizeof(uint)) >>> Log2NibbleSize; protected const uint NibblesPerDwordMask = NibblesPerDword - 1; protected const uint BytesPerBucket = NibblesPerDword * (1 << Log2CodeAlign); @@ -40,12 +40,12 @@ internal abstract class NibbleMapTestBuilderBase protected ulong Addr2Pos(ulong addr) { - return addr >> Log2BytesPerBucket; + return addr >>> Log2BytesPerBucket; } protected uint Addr2Offs(ulong addr) { - return (uint) (((addr & MaskBytesPerBucket) >> Log2CodeAlign) + 1); + return (uint) (((addr & MaskBytesPerBucket) >>> Log2CodeAlign) + 1); } protected int Pos2ShiftCount (ulong addr) @@ -100,8 +100,8 @@ public override void AllocateCodeChunk(TargetCodePointer codeStart, int codeSize bool bSet = true; uint value = bSet?Addr2Offs(delta):0; - uint index = (uint) (pos >> Log2NibblesPerDword); - uint mask = ~(HighestNibbleMask >> (int)((pos & NibblesPerDwordMask) << Log2NibbleSize)); + uint index = (uint) (pos >>> Log2NibblesPerDword); + uint mask = ~(HighestNibbleMask >>> (int)((pos & NibblesPerDwordMask) << Log2NibbleSize)); value = value << Pos2ShiftCount(pos); @@ -142,8 +142,8 @@ public override void AllocateCodeChunk(TargetCodePointer codeStart, int codeSize ulong pos = Addr2Pos(delta); uint value = Addr2Offs(delta); - uint index = (uint) (pos >> Log2NibblesPerDword); - uint mask = ~(HighestNibbleMask >> (int)((pos & NibblesPerDwordMask) << Log2NibbleSize)); + uint index = (uint) (pos >>> Log2NibblesPerDword); + uint mask = ~(HighestNibbleMask >>> (int)((pos & NibblesPerDwordMask) << Log2NibbleSize)); value = value << Pos2ShiftCount(pos); From 8cd0fcf2e37f18e9d0a53e4c223d1729fdd091ad Mon Sep 17 00:00:00 2001 From: Max Charlamb Date: Fri, 8 Nov 2024 11:11:46 -0500 Subject: [PATCH 06/12] comments --- .../Contracts/ExecutionManager/ExecutionManagerFactory.cs | 3 --- .../Contracts/ExecutionManager/ExecutionManager_1.cs | 5 +---- .../Contracts/ExecutionManager/ExecutionManager_2.cs | 5 +---- .../{NibbleMap_2.cs => NibbleMapConstantLookup.cs} | 8 ++++---- .../Helpers/{NibbleMap_1.cs => NibbleMapLinearLookup.cs} | 6 +++--- .../ExecutionManagerTestBuilder.cs | 2 +- .../ExecutionManagerTests.cs | 5 ++--- .../NibbleMapTestBuilder.cs | 8 ++------ .../NibbleMapTests.cs | 6 +++--- .../RangeSectionMapTests.cs | 2 +- 10 files changed, 18 insertions(+), 32 deletions(-) rename src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/{NibbleMap_2.cs => NibbleMapConstantLookup.cs} (95%) rename src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/{NibbleMap_1.cs => NibbleMapLinearLookup.cs} (98%) rename src/native/managed/cdacreader/tests/{ExecutionManagerTests => ExecutionManager}/ExecutionManagerTestBuilder.cs (99%) rename src/native/managed/cdacreader/tests/{ExecutionManagerTests => ExecutionManager}/ExecutionManagerTests.cs (99%) rename src/native/managed/cdacreader/tests/{ExecutionManagerTests => ExecutionManager}/NibbleMapTestBuilder.cs (97%) rename src/native/managed/cdacreader/tests/{ExecutionManagerTests => ExecutionManager}/NibbleMapTests.cs (97%) rename src/native/managed/cdacreader/tests/{ExecutionManagerTests => ExecutionManager}/RangeSectionMapTests.cs (99%) diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerFactory.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerFactory.cs index 4f6aa4477f778c..e99074a2b810fe 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerFactory.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManagerFactory.cs @@ -1,9 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; -using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; - namespace Microsoft.Diagnostics.DataContractReader.Contracts; internal sealed class ExecutionManagerFactory : IContractFactory diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_1.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_1.cs index 8d46c68db75709..cd605240191874 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_1.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_1.cs @@ -1,14 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; namespace Microsoft.Diagnostics.DataContractReader.Contracts; -internal class ExecutionManager_1 : ExecutionManagerBase +internal sealed class ExecutionManager_1 : ExecutionManagerBase { public ExecutionManager_1(Target target, Data.RangeSectionMap topRangeSectionMap) : base(target, topRangeSectionMap) { diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_2.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_2.cs index 620114876a879a..352d4c52b05481 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_2.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_2.cs @@ -1,14 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; namespace Microsoft.Diagnostics.DataContractReader.Contracts; -internal class ExecutionManager_2 : ExecutionManagerBase +internal sealed class ExecutionManager_2 : ExecutionManagerBase { public ExecutionManager_2(Target target, Data.RangeSectionMap topRangeSectionMap) : base(target, topRangeSectionMap) { diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapConstantLookup.cs similarity index 95% rename from src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs rename to src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapConstantLookup.cs index 104480c1e28d87..38a0122cfd7602 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_2.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapConstantLookup.cs @@ -11,7 +11,7 @@ namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; // CoreCLR nibblemap with O(1) lookup time. // -// Implementation very similar to NibbleMap_1, but with the addition of writing relative pointers +// Implementation very similar to NibbleMapLinearLookup, but with the addition of writing relative pointers // into the nibblemap whenever a code block completely covers a DWORD. This allows for O(1) lookup // with the cost of O(n) write time. // @@ -29,11 +29,11 @@ namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; // In this implementation we may "extend" the lookup period of a function several hundred bytes // if there is not another function following it. -internal class NibbleMap_2 : INibbleMap +internal class NibbleMapConstantLookup : INibbleMap { private readonly Target _target; - private NibbleMap_2(Target target) + private NibbleMapConstantLookup(Target target) { _target = target; } @@ -133,7 +133,7 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt public static INibbleMap Create(Target target) { - return new NibbleMap_2(target); + return new NibbleMapConstantLookup(target); } public TargetPointer FindMethodCode(Data.CodeHeapListNode heapListNode, TargetCodePointer jittedCodeAddress) diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_1.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapLinearLookup.cs similarity index 98% rename from src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_1.cs rename to src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapLinearLookup.cs index 6550fa72c220ff..e32d7b2aaf386d 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMap_1.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapLinearLookup.cs @@ -55,11 +55,11 @@ namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; // We will then align the map index to the start of the current map unit (map index 8) and move back to the previous map unit (map index 7) // At that point, we scan backwards for non-zero map units. Since there are none, we return null. -internal class NibbleMap_1 : INibbleMap +internal class NibbleMapLinearLookup : INibbleMap { private readonly Target _target; - private NibbleMap_1(Target target) + private NibbleMapLinearLookup(Target target) { _target = target; } @@ -143,7 +143,7 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt public static INibbleMap Create(Target target) { - return new NibbleMap_1(target); + return new NibbleMapLinearLookup(target); } public TargetPointer FindMethodCode(Data.CodeHeapListNode heapListNode, TargetCodePointer jittedCodeAddress) diff --git a/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTestBuilder.cs b/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTestBuilder.cs similarity index 99% rename from src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTestBuilder.cs rename to src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTestBuilder.cs index d27c1abce0e6cd..d63ec3eac1edbc 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTestBuilder.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTestBuilder.cs @@ -7,7 +7,7 @@ using InteriorMapValue = Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers.RangeSectionMap.InteriorMapValue; -namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManagerTests; +namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManager; internal class ExecutionManagerTestBuilder { diff --git a/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs b/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTests.cs similarity index 99% rename from src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs rename to src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTests.cs index ef48a476ed083a..d2538e4d7301d0 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManagerTests/ExecutionManagerTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTests.cs @@ -3,11 +3,10 @@ using Xunit; -using Microsoft.Diagnostics.DataContractReader.Contracts; using System.Collections.Generic; -using System; +using Microsoft.Diagnostics.DataContractReader.Contracts; -namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManagerTests; +namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManager; public class ExecutionManagerTests { diff --git a/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTestBuilder.cs b/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTestBuilder.cs similarity index 97% rename from src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTestBuilder.cs rename to src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTestBuilder.cs index 6238ccb484e915..4a36e3a4bdf95b 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTestBuilder.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTestBuilder.cs @@ -1,14 +1,10 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Xunit; - -using Microsoft.Diagnostics.DataContractReader.Contracts; -using System.Collections.Generic; using System; using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; -namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManagerTests; +namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManager; internal abstract class NibbleMapTestBuilderBase { @@ -159,7 +155,7 @@ public override void AllocateCodeChunk(TargetCodePointer codeStart, int codeSize TestPlaceholderTarget.WriteToSpan(newValue, Arch.IsLittleEndian, entry); ulong firstByteAfterMethod = delta + (uint)codeSize; - uint encodedPointer = NibbleMap_2.EncodePointer((uint)delta); + uint encodedPointer = NibbleMapConstantLookup.EncodePointer((uint)delta); index++; while((index + 1) * 256 <= firstByteAfterMethod) { diff --git a/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTests.cs b/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs similarity index 97% rename from src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTests.cs rename to src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs index 5b8a4e8cb317fe..aa5c86b07014cf 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManagerTests/NibbleMapTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs @@ -7,7 +7,7 @@ using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; using System.Diagnostics; -namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManagerTests; +namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManager; public class NibbleMapTestsBase { @@ -93,7 +93,7 @@ public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) // TESTCASE: - NibbleMap_1 map = (NibbleMap_1)NibbleMap_1.Create(target); + NibbleMapLinearLookup map = (NibbleMapLinearLookup)NibbleMapLinearLookup.Create(target); Assert.NotNull(map); TargetPointer methodCode = map.FindMethodCode(mapBase, mapStart, inputPC); @@ -153,7 +153,7 @@ public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) // TESTCASE: - NibbleMap_2 map = (NibbleMap_2)NibbleMap_2.Create(target); + NibbleMapConstantLookup map = (NibbleMapConstantLookup)NibbleMapConstantLookup.Create(target); Assert.NotNull(map); TargetPointer methodCode = map.FindMethodCode(mapBase, mapStart, inputPC); diff --git a/src/native/managed/cdacreader/tests/ExecutionManagerTests/RangeSectionMapTests.cs b/src/native/managed/cdacreader/tests/ExecutionManager/RangeSectionMapTests.cs similarity index 99% rename from src/native/managed/cdacreader/tests/ExecutionManagerTests/RangeSectionMapTests.cs rename to src/native/managed/cdacreader/tests/ExecutionManager/RangeSectionMapTests.cs index 32cb330fdd0478..36d3a5f9647d0a 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManagerTests/RangeSectionMapTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/RangeSectionMapTests.cs @@ -7,7 +7,7 @@ using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; -namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManagerTests; +namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManager; public class RangeSectionMapTests { From a471477a3430c8b18867d706f012ec77472b7ad9 Mon Sep 17 00:00:00 2001 From: Max Charlamb Date: Fri, 8 Nov 2024 12:35:22 -0500 Subject: [PATCH 07/12] comments --- docs/design/datacontracts/ExecutionManager.md | 65 +++++++++++++++++-- .../ExecutionManager/ExecutionManager_1.cs | 2 +- .../ExecutionManager/ExecutionManager_2.cs | 2 +- 3 files changed, 63 insertions(+), 6 deletions(-) diff --git a/docs/design/datacontracts/ExecutionManager.md b/docs/design/datacontracts/ExecutionManager.md index 71b8477d90d30b..ba3abb28fd8f32 100644 --- a/docs/design/datacontracts/ExecutionManager.md +++ b/docs/design/datacontracts/ExecutionManager.md @@ -193,12 +193,13 @@ It takes advantage of the fact that the code starts are aligned and are spaced apart to represent their addresses as a 4-bit nibble value. Given a contiguous region of memory in which we lay out a collection of non-overlapping code blocks that are -not too small (so that two adjacent ones aren't too close together) and where the start of each code block is preceeded by a code header aligned on some power of 2, +not too small (so that two adjacent ones aren't too close together) and where the start of each code block is aligned on some power of 2 and preceeded by a code header, we can break up the whole memory space into buckets of a fixed size (32-bytes in the current implementation), where -each bucket either has a code block header or not. -Thinking of each code block header address as a hex number, we can view it as: `[index, offset, zeros]` +each bucket either has a code block or not. +Thinking of each code block address as a hex number, we can view it as: [index, offset] where each index gives us a bucket and the offset gives us the position of the header within the bucket. -We encode each offset into a 4-bit nibble, reserving the special value 0 to mark the places in the map where a method doesn't start. +In the current implementation code must be 4 byte aligned therefore there are 8 possible offsets in a bucket. +These are encoded as values 1-8 in the 4-bit nibble, with 0 reserved to mark the places in the map where a method doesn't start. To find the start of a method given an address we first convert it into a bucket index (giving the map unit) and an offset which we can then turn into the index of the nibble that covers that address. @@ -239,3 +240,59 @@ Now suppose we do a lookup for address 302 (0x12E) * Therefore we know there is no method start at any map index in the current map unit. * We will then align the map index to the start of the current map unit (map index 8) and move back to the previous map unit (map index 7) * At that point, we scan backwards for a non-zero map unit and a non-zero nibble within the first non-zero map unit. Since there are none, we return null. + + +## Version 2 + +Version 2 of the contract uses the new `NibbleMapConstantLookup` algorithm which has O(1) lookup time compared to the `NibbleMapLinearLookup` O(n) lookup time. + +With the exception of the nibblemap change, version 2 is identical to version 1. + +### NibbleMap + +The `NibbleMapConstantLookup` implementation is very similar to `NibbleMapLinearLookup` with the addition +of writing relative pointers into the nibblemap whenever a code block completely covers the code region +represented by a DWORD, with the current values 256 bytes. +This allows for O(1) lookup time with the cost of O(n) write time. + +Pointers are encoded using the top 28 bits of the DWORD as normal. The bottom 4 bits of the pointer +are reduced to 2 bits of data using the fact that code start must be 4 byte aligned. This is encoded into +the nibble in bits 28 .. 31 of the DWORD with values 9-12. This is also used to differentiate DWORDs +filled with nibble values and DWORDs with pointer values. + +| Nibble Value | Meaning | How to decode | +|:------------:|:--------|:--------------:| +| 0 | empty | | +| 1-8 | Nibble | value - 1 | +| 9-12 | Pointer | value - 1 << 2 +| 13-15 | unused | | + +To read the nibblemap, we check if the DWORD is a pointer. If so, then we know the value currentPC is +part of a managed code block beginning at the mapBase + decoded pointer. Otherwise we can check for nibbles +as normal. If the DWORD is empty (no pointer or previous nibbles), then we check the previous DWORD for a +pointer or preceeding nibble. If that DWORD is empty, then we must not be in a managed function. If we were, +the write algorithm would have written a relative pointer in the DWORD or we would have seen the start nibble. + +Note, a currentPC pointing to bytes outside a function has undefined lookup behavior. +In this implementation we may "extend" the lookup period of a function several hundred bytes +if there is not another function immediately following it. + +We will go through the same example as above with the new algorithm. Suppose there is code starting at address 304 (0x130) with length 1024 (0x400). + +* There will be a nibble at the start of the function as before. + * The map index will be 304 / 32 = 9 and the byte offset will be 304 % 32 = 16 + * Because addresses are 4-byte aligned, the nibble value will be 1 + 16 / 4 = 5 (we reserve 0 to mean no method). + * So the map unit containing index 9 will contain the value 0x5 << 24 (the map index 9 means we want the second nibble in the second map unit, and we number the nibbles starting from the most significant) , or 0x05000000 +* Since the function starts at 304 with a length of 1024, the last byte of the function is at 1327 (0x52F). Map units (DWORDs) contain 256 bytes (0x100) algined to the map base. Therefore map units represnting 0x200-0x2FF, 0x300-0x3FF and 0x400-0x4ff are completely covered by the function and will have a relative pointer. + * To get the relative pointer value we split the code start value at the bottom 4 bits. The top 28 bits are included as normal. We shift the bottom 4 bits 2 to the right and add 9, to get the bottom 4 bits encoding. This gives us a relative pointer value of 311 (0x137). + * 304 = 0b100110000 + * Top 28 bits: 304 = 0b10011xxxx + * Bottom 4 bits: 0 = 0b0000 + * Bottom 4 bits encoding: 9 = (0 >> 2) + 9 + * Relative Pointer Encoding: 311 = 304 + 9 + +Now suppose we do a lookup for address 1300 (0x514) +* The map index will be 1300 / 32 = 40 which is located in the 40 / 8 = 5th map unit (DWORD). +* We read the value of the 5th map unit and find it is empty. +* We read the value of the 4th map unit and find that the nibble in the lowest bits has the value of 9 implying that this map unit is a relative pointer. +* Since we found a relative pointer we can decode the entire map unit as a relative pointer and return that address added to the base. diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_1.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_1.cs index cd605240191874..c1a8e135b9d438 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_1.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_1.cs @@ -5,7 +5,7 @@ namespace Microsoft.Diagnostics.DataContractReader.Contracts; -internal sealed class ExecutionManager_1 : ExecutionManagerBase +internal sealed class ExecutionManager_1 : ExecutionManagerBase { public ExecutionManager_1(Target target, Data.RangeSectionMap topRangeSectionMap) : base(target, topRangeSectionMap) { diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_2.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_2.cs index 352d4c52b05481..fc0397d82dfbce 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_2.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/ExecutionManager_2.cs @@ -5,7 +5,7 @@ namespace Microsoft.Diagnostics.DataContractReader.Contracts; -internal sealed class ExecutionManager_2 : ExecutionManagerBase +internal sealed class ExecutionManager_2 : ExecutionManagerBase { public ExecutionManager_2(Target target, Data.RangeSectionMap topRangeSectionMap) : base(target, topRangeSectionMap) { From 9a90bd634c170bcae328a178fbf1a700135508b0 Mon Sep 17 00:00:00 2001 From: Max Charlamb Date: Fri, 8 Nov 2024 15:41:46 -0500 Subject: [PATCH 08/12] fix typos --- .../tests/ExecutionManager/ExecutionManagerTestBuilder.cs | 6 +++--- .../tests/ExecutionManager/ExecutionManagerTests.cs | 2 +- .../cdacreader/tests/ExecutionManager/NibbleMapTests.cs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTestBuilder.cs b/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTestBuilder.cs index d63ec3eac1edbc..0e79a4a70eae44 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTestBuilder.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTestBuilder.cs @@ -174,7 +174,7 @@ public static RangeSectionMapTestBuilder CreateRangeSection(MockTarget.Architect return new RangeSectionMapTestBuilder(arch); } - internal int Verion { get;} + internal int Version { get;} internal MockMemorySpace.Builder Builder { get; } private readonly RangeSectionMapTestBuilder _rsmBuilder; @@ -190,7 +190,7 @@ public static RangeSectionMapTestBuilder CreateRangeSection(MockTarget.Architect internal ExecutionManagerTestBuilder(int version, MockMemorySpace.Builder builder, AllocationRange allocationRange, Dictionary? typeInfoCache = null) { - _version = version; + Version = version; Builder = builder; _rsmBuilder = new RangeSectionMapTestBuilder(ExecutionManagerCodeRangeMapAddress, builder); _rangeSectionMapAllocator = Builder.CreateAllocator(allocationRange.RangeSectionMapStart, allocationRange.RangeSectionMapEnd); @@ -260,7 +260,7 @@ internal static void AddToTypeInfoCache(TargetTestHelpers targetTestHelpers, Dic internal NibbleMapTestBuilderBase CreateNibbleMap(ulong codeRangeStart, uint codeRangeSize) { - NibbleMapTestBuilderBase nibBuilder = _version switch + NibbleMapTestBuilderBase nibBuilder = Version switch { 1 => new NibbleMapTestBuilder_1(codeRangeStart, codeRangeSize, _nibbleMapAllocator, Builder.TargetTestHelpers.Arch), diff --git a/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTests.cs b/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTests.cs index d2538e4d7301d0..b0298c377b8373 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTests.cs @@ -21,7 +21,7 @@ public static ExecutionManagerTestTarget FromBuilder(ExecutionManagerTestBuilder ReadFromTargetDelegate reader = emBuilder.Builder.GetReadContext().ReadFromTarget; var topRangeSectionMap = ExecutionManagerTestBuilder.ExecutionManagerCodeRangeMapAddress; var typeInfo = emBuilder.TypeInfoCache; - return new ExecutionManagerTestTarget(emBuilder.Verion, arch, reader, topRangeSectionMap, typeInfo); + return new ExecutionManagerTestTarget(emBuilder.Version, arch, reader, topRangeSectionMap, typeInfo); } public ExecutionManagerTestTarget(int version, MockTarget.Architecture arch, ReadFromTargetDelegate dataReader, TargetPointer topRangeSectionMap, Dictionary typeInfoCache) : base(arch) diff --git a/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs b/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs index aa5c86b07014cf..f4bd34fb74eb79 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs @@ -60,7 +60,7 @@ public void ExhaustiveNibbbleShifts(ulong irrelevant) for (int i = 0; i < 255; i++) { NibbleMapHelpers.MapKey input = new (irrelevant + (ulong)i); - int actualShift = NibbleMapHelpers.ComputeNibbleShift(input); + uint actualShift = NibbleMapHelpers.ComputeNibbleShift(input); Assert.True(expectedShift == actualShift, $"Expected {expectedShift}, got {actualShift} for input {input}"); expectedShift -= 4; if (expectedShift == -4) From 8218390b7d1e2ef270a15a8a963bc7e57dbf88d9 Mon Sep 17 00:00:00 2001 From: Max Charlamb Date: Fri, 8 Nov 2024 15:46:59 -0500 Subject: [PATCH 09/12] rename --- .../cdacreader/tests/ExecutionManager/NibbleMapTests.cs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs b/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs index f4bd34fb74eb79..2386cf16a76cb6 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs @@ -1,11 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using Xunit; using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; -using System.Diagnostics; namespace Microsoft.Diagnostics.DataContractReader.UnitTests.ExecutionManager; @@ -30,7 +28,7 @@ internal static NibbleMapTestTarget CreateTarget(NibbleMapTestBuilderBase nibble } } -public class NibbleMapTests_1 : NibbleMapTestsBase +public class NibbleMapLinearLookupTests : NibbleMapTestsBase { [Fact] public void RoundTripAddressTest() @@ -128,7 +126,7 @@ public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) } } -public class NibbleMapTests_2 : NibbleMapTestsBase +public class NibbleMapConstantLookupTests : NibbleMapTestsBase { [Theory] [ClassData(typeof(MockTarget.StdArch))] From e358fe7c3e7c355ea25a3b63db46aad05eb15812 Mon Sep 17 00:00:00 2001 From: Max Charlamb Date: Mon, 11 Nov 2024 14:25:19 -0500 Subject: [PATCH 10/12] comments --- .../Helpers/NibbleMapConstantLookup.cs | 29 +++++------ .../ExecutionManagerTestBuilder.cs | 6 +-- .../ExecutionManager/ExecutionManagerTests.cs | 50 +++++++++++++++++-- .../ExecutionManager/NibbleMapTestBuilder.cs | 6 +-- .../tests/ExecutionManager/NibbleMapTests.cs | 4 +- 5 files changed, 68 insertions(+), 27 deletions(-) diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapConstantLookup.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapConstantLookup.cs index 38a0122cfd7602..bb18f4584d8f2c 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapConstantLookup.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/ExecutionManager/Helpers/NibbleMapConstantLookup.cs @@ -110,25 +110,26 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt // read the map unit containing mapIdx and skip over it if it is all zeros t = mapIdx.ReadMapUnit(_target, mapStart); - // if t is not zero, we either have a pointer or a nibble - if (!t.IsEmpty) + // if t is empty, then currentPC can not be in a function + if (t.IsEmpty) { - if (IsPointer(t)) - { - return DecodePointer(mapBase, t); - } + return TargetPointer.Null; + } - // move to the correct nibble in the map unit - while (!mapIdx.IsZero && t.Nibble.IsEmpty) - { - t = t.ShiftNextNibble; - mapIdx = mapIdx.Prev; - } + // if t is not empty, it must contain a pointer or a nibble + if (IsPointer(t)) + { + return DecodePointer(mapBase, t); + } - return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); + // move to the correct nibble in the map unit + while (!mapIdx.IsZero && t.Nibble.IsEmpty) + { + t = t.ShiftNextNibble; + mapIdx = mapIdx.Prev; } - return TargetPointer.Null; + return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); } public static INibbleMap Create(Target target) diff --git a/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTestBuilder.cs b/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTestBuilder.cs index 0e79a4a70eae44..6a3e32dd49cb66 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTestBuilder.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTestBuilder.cs @@ -345,16 +345,16 @@ public TargetPointer AddCodeHeapListNode(TargetPointer next, TargetPointer start // offset from the start of the code private uint CodeHeaderOffset => CodeHeaderSize; - private (MockMemorySpace.HeapFragment fragment, TargetCodePointer codeStart) AllocateJittedMethod(JittedCodeRange jittedCodeRange, int codeSize, string name = "Method Header & Code") + private (MockMemorySpace.HeapFragment fragment, TargetCodePointer codeStart) AllocateJittedMethod(JittedCodeRange jittedCodeRange, uint codeSize, string name = "Method Header & Code") { - ulong size = (ulong)(codeSize + CodeHeaderOffset); + ulong size = codeSize + CodeHeaderOffset; MockMemorySpace.HeapFragment methodFragment = jittedCodeRange.Allocator.Allocate(size, name); Builder.AddHeapFragment(methodFragment); TargetCodePointer codeStart = methodFragment.Address + CodeHeaderOffset; return (methodFragment, codeStart); } - public TargetCodePointer AddJittedMethod(JittedCodeRange jittedCodeRange, int codeSize, TargetPointer methodDescAddress) + public TargetCodePointer AddJittedMethod(JittedCodeRange jittedCodeRange, uint codeSize, TargetPointer methodDescAddress) { (MockMemorySpace.HeapFragment methodFragment, TargetCodePointer codeStart) = AllocateJittedMethod(jittedCodeRange, codeSize); diff --git a/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTests.cs b/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTests.cs index b0298c377b8373..df151ee05dcac6 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/ExecutionManagerTests.cs @@ -97,7 +97,7 @@ public void LookupNonNullOneRangeOneMethod(int version, MockTarget.Architecture { const ulong codeRangeStart = 0x0a0a_0000u; // arbitrary const uint codeRangeSize = 0xc000u; // arbitrary - int methodSize = 0x450; // arbitrary + const uint methodSize = 0x450; // arbitrary TargetPointer jitManagerAddress = new (0x000b_ff00); // arbitrary @@ -119,26 +119,66 @@ public void LookupNonNullOneRangeOneMethod(int version, MockTarget.Architecture var target = ExecutionManagerTestTarget.FromBuilder(emBuilder); - // test at method start - var em = target.Contracts.ExecutionManager; Assert.NotNull(em); + + // test at method start var eeInfo = em.GetCodeBlockHandle(methodStart); Assert.NotNull(eeInfo); TargetPointer actualMethodDesc = em.GetMethodDesc(eeInfo.Value); Assert.Equal(expectedMethodDescAddress, actualMethodDesc); // test middle of method - eeInfo = em.GetCodeBlockHandle(methodStart + 0x250); + eeInfo = em.GetCodeBlockHandle(methodStart + methodSize / 2); Assert.NotNull(eeInfo); + actualMethodDesc = em.GetMethodDesc(eeInfo.Value); Assert.Equal(expectedMethodDescAddress, actualMethodDesc); // test end of method - eeInfo = em.GetCodeBlockHandle(methodStart + 0x450 - 1); + eeInfo = em.GetCodeBlockHandle(methodStart + methodSize - 1); Assert.NotNull(eeInfo); + actualMethodDesc = em.GetMethodDesc(eeInfo.Value); Assert.Equal(expectedMethodDescAddress, actualMethodDesc); } + [Theory] + [MemberData(nameof(StdArchAllVersions))] + public void LookupNullOneRangeZeroMethod(int version, MockTarget.Architecture arch) + { + const ulong codeRangeStart = 0x0a0a_0000u; // arbitrary + const uint codeRangeSize = 0xc000u; // arbitrary + + TargetPointer jitManagerAddress = new (0x000b_ff00); // arbitrary + + ExecutionManagerTestBuilder emBuilder = new(version, arch, ExecutionManagerTestBuilder.DefaultAllocationRange); + var jittedCode = emBuilder.AllocateJittedCodeRange(codeRangeStart, codeRangeSize); + + NibbleMapTestBuilderBase nibBuilder = emBuilder.CreateNibbleMap(codeRangeStart, codeRangeSize); + + TargetPointer codeHeapListNodeAddress = emBuilder.AddCodeHeapListNode(TargetPointer.Null, codeRangeStart, codeRangeStart + codeRangeSize, codeRangeStart, nibBuilder.NibbleMapFragment.Address); + TargetPointer rangeSectionAddress = emBuilder.AddRangeSection(jittedCode, jitManagerAddress: jitManagerAddress, codeHeapListNodeAddress: codeHeapListNodeAddress); + TargetPointer rangeSectionFragmentAddress = emBuilder.AddRangeSectionFragment(jittedCode, rangeSectionAddress); + + emBuilder.MarkCreated(); + + var target = ExecutionManagerTestTarget.FromBuilder(emBuilder); + + var em = target.Contracts.ExecutionManager; + Assert.NotNull(em); + + // test at code range start + var eeInfo = em.GetCodeBlockHandle(codeRangeSize + codeRangeSize); + Assert.Null(eeInfo); + + // test middle of code range + eeInfo = em.GetCodeBlockHandle(codeRangeSize + codeRangeSize / 2); + Assert.Null(eeInfo); + + // test end of code range + eeInfo = em.GetCodeBlockHandle(codeRangeSize + codeRangeSize - 1); + Assert.Null(eeInfo); + } + public static IEnumerable StdArchAllVersions() { const int highestVersion = 2; diff --git a/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTestBuilder.cs b/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTestBuilder.cs index 4a36e3a4bdf95b..3e51e6f890cecd 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTestBuilder.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTestBuilder.cs @@ -69,7 +69,7 @@ public NibbleMapTestBuilderBase(TargetPointer mapBase, ulong mapRangeSize, MockM NibbleMapFragment = allocator.Allocate((ulong)nibbleMapSize, "Nibble Map"); } - public abstract void AllocateCodeChunk(TargetCodePointer codeStart, int codeSize); + public abstract void AllocateCodeChunk(TargetCodePointer codeStart, uint codeSize); } internal class NibbleMapTestBuilder_1 : NibbleMapTestBuilderBase @@ -84,7 +84,7 @@ public NibbleMapTestBuilder_1(TargetPointer mapBase, ulong mapRangeSize, MockMem { } - public override void AllocateCodeChunk(TargetCodePointer codeStart, int codeSize) + public override void AllocateCodeChunk(TargetCodePointer codeStart, uint codeSize) { // paraphrased from EEJitManager::NibbleMapSetUnlocked if (codeStart.Value < MapBase.Value) @@ -126,7 +126,7 @@ public NibbleMapTestBuilder_2(TargetPointer mapBase, ulong mapRangeSize, MockMem { } - public override void AllocateCodeChunk(TargetCodePointer codeStart, int codeSize) + public override void AllocateCodeChunk(TargetCodePointer codeStart, uint codeSize) { // paraphrased from EEJitManager::NibbleMapSetUnlocked if (codeStart.Value < MapBase.Value) diff --git a/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs b/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs index 2386cf16a76cb6..c92fc3f1afa4ce 100644 --- a/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs +++ b/src/native/managed/cdacreader/tests/ExecutionManager/NibbleMapTests.cs @@ -85,7 +85,7 @@ public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) // don't put the code too close to the start - the NibbleMap bails if the code is too close to the start of the range TargetCodePointer inputPC = new(mapBase + 0x0200u); - int codeSize = 0x80; // doesn't matter + uint codeSize = 0x80; // doesn't matter builder.AllocateCodeChunk (inputPC, codeSize); NibbleMapTestTarget target = CreateTarget(builder); @@ -145,7 +145,7 @@ public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) // don't put the code too close to the start - the NibbleMap bails if the code is too close to the start of the range TargetCodePointer inputPC = new(mapBase + 0x0200u); - int codeSize = 0x400; + uint codeSize = 0x400; builder.AllocateCodeChunk (inputPC, codeSize); NibbleMapTestTarget target = CreateTarget(builder); From 184140579deb4d5d100e8434dd650fdb9d5c5c49 Mon Sep 17 00:00:00 2001 From: Max Charlamb Date: Mon, 11 Nov 2024 14:26:48 -0500 Subject: [PATCH 11/12] doc update --- docs/design/datacontracts/ExecutionManager.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/design/datacontracts/ExecutionManager.md b/docs/design/datacontracts/ExecutionManager.md index ba3abb28fd8f32..fd9869b6722bae 100644 --- a/docs/design/datacontracts/ExecutionManager.md +++ b/docs/design/datacontracts/ExecutionManager.md @@ -186,12 +186,14 @@ code allocated in that address range), level 4 entires point to level 3 maps and ### NibbleMap -Version 1 of this contract depends on a "nibble map" data structure +The ExecutionManager contract depends on a "nibble map" data structure that allows mapping of a code address in a contiguous subsection of the address space to the pointer to the start of that a code sequence. It takes advantage of the fact that the code starts are aligned and are spaced apart to represent their addresses as a 4-bit nibble value. +Version 1 of the contract depends on the `NibbleMapLinearLookup` implementation of the nibblemap algorithm. + Given a contiguous region of memory in which we lay out a collection of non-overlapping code blocks that are not too small (so that two adjacent ones aren't too close together) and where the start of each code block is aligned on some power of 2 and preceeded by a code header, we can break up the whole memory space into buckets of a fixed size (32-bytes in the current implementation), where @@ -244,7 +246,7 @@ Now suppose we do a lookup for address 302 (0x12E) ## Version 2 -Version 2 of the contract uses the new `NibbleMapConstantLookup` algorithm which has O(1) lookup time compared to the `NibbleMapLinearLookup` O(n) lookup time. +Version 2 of the contract depends the new `NibbleMapConstantLookup` algorithm which has O(1) lookup time compared to the `NibbleMapLinearLookup` O(n) lookup time. With the exception of the nibblemap change, version 2 is identical to version 1. @@ -255,7 +257,7 @@ of writing relative pointers into the nibblemap whenever a code block completely represented by a DWORD, with the current values 256 bytes. This allows for O(1) lookup time with the cost of O(n) write time. -Pointers are encoded using the top 28 bits of the DWORD as normal. The bottom 4 bits of the pointer +Pointers are encoded using the top 28 bits of the DWORD. The bottom 4 bits of the pointer are reduced to 2 bits of data using the fact that code start must be 4 byte aligned. This is encoded into the nibble in bits 28 .. 31 of the DWORD with values 9-12. This is also used to differentiate DWORDs filled with nibble values and DWORDs with pointer values. @@ -264,16 +266,16 @@ filled with nibble values and DWORDs with pointer values. |:------------:|:--------|:--------------:| | 0 | empty | | | 1-8 | Nibble | value - 1 | -| 9-12 | Pointer | value - 1 << 2 +| 9-12 | Pointer | (value - 9) << 2 | | 13-15 | unused | | -To read the nibblemap, we check if the DWORD is a pointer. If so, then we know the value currentPC is -part of a managed code block beginning at the mapBase + decoded pointer. Otherwise we can check for nibbles +To read the nibblemap, we check if the DWORD is a pointer. If so, then we know the value looked is +part of a managed code block beginning at the map base + decoded pointer. Otherwise we can check for nibbles as normal. If the DWORD is empty (no pointer or previous nibbles), then we check the previous DWORD for a pointer or preceeding nibble. If that DWORD is empty, then we must not be in a managed function. If we were, the write algorithm would have written a relative pointer in the DWORD or we would have seen the start nibble. -Note, a currentPC pointing to bytes outside a function has undefined lookup behavior. +Note, looking up a value that points to bytes outside of a managed function has undefined behavior. In this implementation we may "extend" the lookup period of a function several hundred bytes if there is not another function immediately following it. From 0f01f80ade1e71f4e3f00b7979b3116e2238128a Mon Sep 17 00:00:00 2001 From: Max Charlamb Date: Mon, 11 Nov 2024 14:29:26 -0500 Subject: [PATCH 12/12] typo --- docs/design/datacontracts/ExecutionManager.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/datacontracts/ExecutionManager.md b/docs/design/datacontracts/ExecutionManager.md index fd9869b6722bae..ea032d3cec32cf 100644 --- a/docs/design/datacontracts/ExecutionManager.md +++ b/docs/design/datacontracts/ExecutionManager.md @@ -269,7 +269,7 @@ filled with nibble values and DWORDs with pointer values. | 9-12 | Pointer | (value - 9) << 2 | | 13-15 | unused | | -To read the nibblemap, we check if the DWORD is a pointer. If so, then we know the value looked is +To read the nibblemap, we check if the DWORD is a pointer. If so, then we know the value looked up is part of a managed code block beginning at the map base + decoded pointer. Otherwise we can check for nibbles as normal. If the DWORD is empty (no pointer or previous nibbles), then we check the previous DWORD for a pointer or preceeding nibble. If that DWORD is empty, then we must not be in a managed function. If we were,