Prevent devirtualization into unallocated types

src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ILScanner.cs

@@ -365,8 +365,8 @@ public override DictionaryLayoutNode GetLayout(TypeSystemEntity methodOrType)
         private class ScannedDevirtualizationManager : DevirtualizationManager
         {
             private HashSet<TypeDesc> _constructedTypes = new HashSet<TypeDesc>();
+            private HashSet<TypeDesc> _canonConstructedTypes = new HashSet<TypeDesc>();
             private HashSet<TypeDesc> _unsealedTypes = new HashSet<TypeDesc>();
-            private HashSet<TypeDesc> _abstractButNonabstractlyOverriddenTypes = new HashSet<TypeDesc>();
 
             public ScannedDevirtualizationManager(ImmutableArray<DependencyNodeCore<NodeFactory>> markedNodes)
             {
@@ -390,16 +390,13 @@ public ScannedDevirtualizationManager(ImmutableArray<DependencyNodeCore<NodeFact
                             // 2. What types are the base types of other types
                             //    This is needed for optimizations. We use this information to effectively
                             //    seal types that are not base types for any other type.
-                            // 3. What abstract types got derived by non-abstract types.
-                            //    This is needed for correctness. Abstract types that were never derived
-                            //    by non-abstract types should never be devirtualized into - we probably
-                            //    didn't scan the virtual methods on them.
                             //
 
                             if (!type.IsCanonicalSubtype(CanonicalFormKind.Any))
                                 _constructedTypes.Add(type);
 
                             TypeDesc canonType = type.ConvertToCanonForm(CanonicalFormKind.Specific);
+                            _canonConstructedTypes.Add(canonType.GetClosestDefType());
 
                             bool hasNonAbstractTypeInHierarchy = canonType is not MetadataType mdType || !mdType.IsAbstract;
                             TypeDesc baseType = canonType.BaseType;
@@ -408,12 +405,6 @@ public ScannedDevirtualizationManager(ImmutableArray<DependencyNodeCore<NodeFact
                             {
                                 baseType = baseType.ConvertToCanonForm(CanonicalFormKind.Specific);
                                 added = _unsealedTypes.Add(baseType);
-
-                                bool currentTypeIsAbstract = ((MetadataType)baseType).IsAbstract;
-                                if (currentTypeIsAbstract && hasNonAbstractTypeInHierarchy)
-                                    added |= _abstractButNonabstractlyOverriddenTypes.Add(baseType);
-                                hasNonAbstractTypeInHierarchy |= !currentTypeIsAbstract;
-
                                 baseType = baseType.BaseType;
                             }
                         }
@@ -449,15 +440,11 @@ public override bool IsEffectivelySealed(TypeDesc type)
             protected override MethodDesc ResolveVirtualMethod(MethodDesc declMethod, DefType implType, out CORINFO_DEVIRTUALIZATION_DETAIL devirtualizationDetail)
             {
                 MethodDesc result = base.ResolveVirtualMethod(declMethod, implType, out devirtualizationDetail);
-                if (result != null && result.IsFinal && result.OwningType is MetadataType mdType && mdType.IsAbstract)
+                if (result != null)
                 {
-                    // If this type is abstract check that we saw a non-abstract type deriving from it.
-                    // We don't look at virtual methods introduced by abstract classes unless there's a non-abstract
-                    // class that needs them (i.e. the non-abstract class doesn't immediately override them).
-                    // This lets us optimize out some unused virtual method implementations.
-                    // Allowing this to devirtualize would cause trouble because we didn't scan the method
-                    // and expected it would be optimized out.
-                    if (!_abstractButNonabstractlyOverriddenTypes.Contains(mdType.ConvertToCanonForm(CanonicalFormKind.Specific)))
+                    // If we would resolve into a type that wasn't seen as allocated, don't allow devirtualization.
+                    // It would go past what we scanned in the scanner and that doesn't lead to good things.
+                    if (!_canonConstructedTypes.Contains(result.OwningType.ConvertToCanonForm(CanonicalFormKind.Specific)))
                     {
                         // FAILED_BUBBLE_IMPL_NOT_REFERENCEABLE is close enough...
                         devirtualizationDetail = CORINFO_DEVIRTUALIZATION_DETAIL.CORINFO_DEVIRTUALIZATION_FAILED_BUBBLE_IMPL_NOT_REFERENCEABLE;

src/libraries/tests.proj

@@ -490,6 +490,7 @@
     <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Net.Mail\tests\Functional\System.Net.Mail.Functional.Tests.csproj"
                       Condition="'$(TargetOS)' == 'windows'"/>
 
+    <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Reflection.Metadata\tests\System.Reflection.Metadata.Tests.csproj" />
     <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Net.Http.Json\tests\FunctionalTests\System.Net.Http.Json.Functional.Tests.csproj" />
     <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Linq.Parallel\tests\System.Linq.Parallel.Tests.csproj" />
     <ProjectExclusions Include="$(MSBuildThisFileDirectory)System.Text.Json\tests\System.Text.Json.Tests\System.Text.Json.Tests.csproj" />

src/tests/nativeaot/SmokeTests/UnitTests/Devirtualization.cs

@@ -12,6 +12,7 @@ internal static int Run()
     {
         RegressionBug73076.Run();
         DevirtualizationCornerCaseTests.Run();
+        DevirtualizeIntoUnallocatedGenericType.Run();
 
         return 100;
     }
@@ -111,4 +112,28 @@ public static void Run()
             TestIntf2((IIntf2)Activator.CreateInstance(typeof(Intf2Impl2<>).MakeGenericType(typeof(object))), 456);
         }
     }
+
+    class DevirtualizeIntoUnallocatedGenericType
+    {
+        class Never { }
+
+        class SomeGeneric<T>
+        {
+            public virtual object GrabObject() => null;
+        }
+
+        sealed class SomeUnallocatedClass<T> : SomeGeneric<T>
+        {
+            public override object GrabObject() => new Never();
+        }
+
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        static SomeUnallocatedClass<object> GrabInst() => null;
+
+        public static void Run()
+        {
+            if (GrabInst() != null)
+                GrabInst().GrabObject();
+        }
+    }
 }