[cDAC] Implement SetCompilerFlags for cDAC

docs/design/datacontracts/Loader.md

@@ -119,6 +119,7 @@ enum DebuggerAssemblyControlFlags : uint
     DACF_NONE = 0x00,
     DACF_ALLOW_JIT_OPTS = 0x02,
     DACF_ENC_ENABLED = 0x08,
+    DACF_IGNORE_PDBS = 0x20,
     DACF_CONTROL_FLAGS_MASK = 0x2E,
 }
 ```

docs/design/datacontracts/ReJIT.md

@@ -29,6 +29,8 @@ Data descriptors used:
 | --- | --- | --- |
 | ProfControlBlock | GlobalEventMask | an `ICorProfiler` `COR_PRF_MONITOR` value |
 | ProfControlBlock | RejitOnAttachEnabled | cached value of the `ProfAPI_RejitOnAttach` configuration knob |
+| ProfControlBlock | MainProfilerProfInterface | pointer to the main profiler's `ICorProfilerCallback` interface, non-null means a main profiler is attached |
+| ProfControlBlock | NotificationProfilerCount | number of notification-only profilers currently attached |
 | ILCodeVersionNode | VersionId | `ILCodeVersion` ReJIT ID
 | ILCodeVersionNode | RejitState | a `RejitFlags` value |
 

src/coreclr/inc/cordbpriv.h

@@ -49,7 +49,7 @@ enum DebuggerControlFlag
 // Flags used to control the debuggable state of modules and
 // assemblies.
 //
-// [cDAC] [Loader]: Contract depends on DACF_NONE, DACF_ALLOW_JIT_OPTS, DACF_ENC_ENABLED, DACF_CONTROL_FLAGS_MASK.
+// [cDAC] [Loader]: Contract depends on DACF_NONE, DACF_ALLOW_JIT_OPTS, DACF_ENC_ENABLED, DACF_IGNORE_PDBS, DACF_CONTROL_FLAGS_MASK.
 enum DebuggerAssemblyControlFlags
 {
     DACF_NONE                       = 0x00,

src/coreclr/vm/datadescriptor/datadescriptor.inc

@@ -924,6 +924,8 @@ CDAC_TYPE_END(ILCodeVersionNode)
 CDAC_TYPE_BEGIN(ProfControlBlock)
 CDAC_TYPE_FIELD(ProfControlBlock, T_UINT64, GlobalEventMask, offsetof(ProfControlBlock, globalEventMask))
 CDAC_TYPE_FIELD(ProfControlBlock, T_BOOL, RejitOnAttachEnabled, offsetof(ProfControlBlock, fRejitOnAttachEnabled))
+CDAC_TYPE_FIELD(ProfControlBlock, T_POINTER, MainProfilerProfInterface, offsetof(ProfControlBlock, mainProfilerInfo) + offsetof(ProfilerInfo, pProfInterface))
+CDAC_TYPE_FIELD(ProfControlBlock, T_INT32, NotificationProfilerCount, offsetof(ProfControlBlock, notificationProfilerCount))
 CDAC_TYPE_END(ProfControlBlock)
 #endif // PROFILING_SUPPORTED
 

src/native/managed/cdac/Microsoft.Diagnostics.DataContractReader.Abstractions/Contracts/ILoader.cs

@@ -33,6 +33,7 @@ public enum DebuggerAssemblyControlFlags : uint
     DACF_NONE = 0x00,
     DACF_ALLOW_JIT_OPTS = 0x02,
     DACF_ENC_ENABLED = 0x08,
+    DACF_IGNORE_PDBS = 0x20,
     DACF_CONTROL_FLAGS_MASK = 0x2E,
 }
 

src/native/managed/cdac/Microsoft.Diagnostics.DataContractReader.Abstractions/CorDbHResults.cs

@@ -10,4 +10,5 @@ public static class CorDbgHResults
     public const int CORDBG_E_READVIRTUAL_FAILURE = unchecked((int)0x80131c49);
     public const int ERROR_BUFFER_OVERFLOW = unchecked((int)0x8007006F); // HRESULT_FROM_WIN32(ERROR_BUFFER_OVERFLOW)
     public const int CORDBG_E_CLASS_NOT_LOADED = unchecked((int)0x80131303);
+    public const int CORDBG_S_NOT_ALL_BITS_SET = unchecked((int)0x00131c13);
 }

src/native/managed/cdac/Microsoft.Diagnostics.DataContractReader.Contracts/Contracts/Loader_1.cs

@@ -23,6 +23,7 @@ private enum ModuleFlags_1 : uint
         JitOptimizationDisabled = 0x2,  // Cached flag: JIT optimizations are disabled
         EditAndContinue = 0x8,          // Edit and Continue is enabled for this module
         ReflectionEmit = 0x40,          // Reflection.Emit was used to create this module
+        EncCapable = 0x200,             // Cached flag: module is Edit and Continue capable
     }
 
     private const uint DebuggerInfoMask = 0x0000FC00;
@@ -389,6 +390,8 @@ private static ModuleFlags GetFlags(Data.Module module)
             flags |= ModuleFlags.EditAndContinue;
         if (runtimeFlags.HasFlag(ModuleFlags_1.ReflectionEmit))
             flags |= ModuleFlags.ReflectionEmit;
+        if (runtimeFlags.HasFlag(ModuleFlags_1.EncCapable))
+            flags |= ModuleFlags.EncCapable;
 
         return flags;
     }

src/native/managed/cdac/Microsoft.Diagnostics.DataContractReader.Contracts/Data/ProfControlBlock.cs

@@ -13,8 +13,12 @@ public ProfControlBlock(Target target, TargetPointer address)
         Target.TypeInfo type = target.GetTypeInfo(DataType.ProfControlBlock);
         GlobalEventMask = target.ReadField<ulong>(address, type, nameof(GlobalEventMask));
         RejitOnAttachEnabled = target.ReadField<byte>(address, type, nameof(RejitOnAttachEnabled)) != 0;
+        MainProfilerProfInterface = target.ReadPointerField(address, type, nameof(MainProfilerProfInterface));
+        NotificationProfilerCount = target.ReadField<int>(address, type, nameof(NotificationProfilerCount));
     }
 
     public ulong GlobalEventMask { get; init; }
     public bool RejitOnAttachEnabled { get; init; }
+    public TargetPointer MainProfilerProfInterface { get; init; }
+    public int NotificationProfilerCount { get; init; }
 }

src/native/managed/cdac/Microsoft.Diagnostics.DataContractReader.Legacy/Dbi/DacDbiImpl.cs

@@ -8,6 +8,7 @@
 using System.Runtime.InteropServices;
 using System.Runtime.InteropServices.Marshalling;
 using Microsoft.Diagnostics.DataContractReader.Contracts;
+using DACF = Microsoft.Diagnostics.DataContractReader.Contracts.DebuggerAssemblyControlFlags;
 
 namespace Microsoft.Diagnostics.DataContractReader.Legacy;
 
@@ -37,6 +38,17 @@ private int StringHolderAssignCopy(nint stringHolder, string str)
         }
     }
 
+    private bool CORProfilerPresent()
+    {
+        if (!_target.TryReadGlobalPointer(Constants.Globals.ProfilerControlBlock, out TargetPointer? profControlBlockAddress))
+            return false;
+
+        Target.TypeInfo type = _target.GetTypeInfo(DataType.ProfControlBlock);
+        TargetPointer mainProfInterface = _target.ReadPointerField(profControlBlockAddress.Value, type, "MainProfilerProfInterface");
+        int notificationCount = _target.ReadField<int>(profControlBlockAddress.Value, type, "NotificationProfilerCount");
+        return mainProfInterface != TargetPointer.Null || notificationCount > 0;
+    }
+
     public DacDbiImpl(Target target, object? legacyObj)
     {
         _target = target;
@@ -315,7 +327,51 @@ public int GetCompilerFlags(ulong vmAssembly, Interop.BOOL* pfAllowJITOpts, Inte
     }
 
     public int SetCompilerFlags(ulong vmAssembly, Interop.BOOL fAllowJitOpts, Interop.BOOL fEnableEnC)
-        => LegacyFallbackHelper.CanFallback() && _legacy is not null ? _legacy.SetCompilerFlags(vmAssembly, fAllowJitOpts, fEnableEnC) : HResults.E_NOTIMPL;
+    {
+        int hr = HResults.S_OK;
+        try
+        {
+            Contracts.ILoader loader = _target.Contracts.Loader;
+            Contracts.ModuleHandle handle = loader.GetModuleHandleFromAssemblyPtr(new TargetPointer(vmAssembly));
+
+            DACF debuggerInfoBits = loader.GetDebuggerInfoBits(handle);
+            DACF controlFlags = debuggerInfoBits & ~(DACF.DACF_ALLOW_JIT_OPTS | DACF.DACF_ENC_ENABLED);
+            controlFlags &= DACF.DACF_CONTROL_FLAGS_MASK;
+
+            if (fAllowJitOpts != Interop.BOOL.FALSE)
+            {
+                controlFlags |= DACF.DACF_ALLOW_JIT_OPTS;
+            }
+
+            if (fEnableEnC != Interop.BOOL.FALSE)
+            {
+                bool fIgnorePdbs = (debuggerInfoBits & DACF.DACF_IGNORE_PDBS) != 0;
+                bool canSetEnC = (loader.GetFlags(handle) & Contracts.ModuleFlags.EncCapable) != 0 && !CORProfilerPresent() && fIgnorePdbs;
+                if (canSetEnC)
+                {
+                    controlFlags |= DACF.DACF_ENC_ENABLED;
+                }
+                else
+                {
+                    hr = CorDbgHResults.CORDBG_S_NOT_ALL_BITS_SET;
+                }
+            }
+
+            loader.SetDebuggerInfoBits(handle, controlFlags);
+        }
+        catch (System.Exception ex)
+        {
+            hr = ex.HResult;
+        }
+#if DEBUG
+        if (_legacy is not null)
+        {
+            int hrLocal = _legacy.SetCompilerFlags(vmAssembly, fAllowJitOpts, fEnableEnC);
+            Debug.ValidateHResult(hr, hrLocal);
+        }
+#endif
+        return hr;
+    }
 
     public int EnumerateAssembliesInAppDomain(ulong vmAppDomain, nint fpCallback, nint pUserData)
         => LegacyFallbackHelper.CanFallback() && _legacy is not null ? _legacy.EnumerateAssembliesInAppDomain(vmAppDomain, fpCallback, pUserData) : HResults.E_NOTIMPL;