From cea0986899baac86c344733bff2c08228368b54b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20Strehovsk=C3=BD?= Date: Sun, 14 Aug 2022 16:26:55 +0900 Subject: [PATCH] Add generation of structured object file dumps The compiler can currently emit an XML file with everything that was emitted into the output object file. This works and it's easily human readable. But the captured information is not very structured and cannot be used to generate aggregate information like 'this assembly contributed X bytes". This adds another dumper format similar to what we do for MIBC and others: it's ECMA-335 based and uses ldtoken/ldstr/ldc to encode the data. This can be used to build better tools. There's currently none but ILDASM, but it would be good to have it for 7.0. Good hackathon project. --- .../Microsoft.NETCore.Native.targets | 1 + .../MethodExceptionHandlingInfoNode.cs | 2 + .../Compiler/MstatObjectDumper.cs | 117 ++++++++++++++++++ .../Compiler/ObjectDumper.cs | 93 ++++++-------- .../Compiler/XmlObjectDumper.cs | 83 +++++++++++++ .../ILCompiler.Compiler.csproj | 5 + src/coreclr/tools/aot/ILCompiler/Program.cs | 12 +- 7 files changed, 253 insertions(+), 60 deletions(-) create mode 100644 src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/MstatObjectDumper.cs create mode 100644 src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/XmlObjectDumper.cs diff --git a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets index d314741a634586..acd1232752009a 100644 --- a/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets +++ b/src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets @@ -213,6 +213,7 @@ The .NET Foundation licenses this file to you under the MIT license. + diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/MethodExceptionHandlingInfoNode.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/MethodExceptionHandlingInfoNode.cs index 3e24857b2cd46a..d2f867d5ee296a 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/MethodExceptionHandlingInfoNode.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/DependencyAnalysis/MethodExceptionHandlingInfoNode.cs @@ -14,6 +14,8 @@ public class MethodExceptionHandlingInfoNode : ObjectNode, ISymbolDefinitionNode private readonly MethodDesc _owningMethod; private readonly ObjectData _data; + public MethodDesc Method => _owningMethod; + public MethodExceptionHandlingInfoNode(MethodDesc owningMethod, ObjectData data) { _owningMethod = owningMethod; diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/MstatObjectDumper.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/MstatObjectDumper.cs new file mode 100644 index 00000000000000..0f0af381c2e6ed --- /dev/null +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/MstatObjectDumper.cs @@ -0,0 +1,117 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Reflection.Metadata; +using System.Reflection.Metadata.Ecma335; + +using Internal.Text; +using Internal.TypeSystem; + +using ILCompiler.DependencyAnalysis; + +using ObjectData = ILCompiler.DependencyAnalysis.ObjectNode.ObjectData; +using AssemblyName = System.Reflection.AssemblyName; +using System.Collections.Generic; +using static ILCompiler.DependencyAnalysis.ObjectNode; + +namespace ILCompiler +{ + public class MstatObjectDumper : ObjectDumper + { + private const int VersionMajor = 1; + private const int VersionMinor = 0; + + private readonly string _fileName; + private readonly TypeSystemMetadataEmitter _emitter; + + private readonly InstructionEncoder _types = new InstructionEncoder(new BlobBuilder()); + + private Dictionary _methods = new(); + private Dictionary _methodEhInfo = new(); + private Dictionary _blobs = new(); + + private Utf8StringBuilder _utf8StringBuilder = new Utf8StringBuilder(); + + public MstatObjectDumper(string fileName, TypeSystemContext context) + { + _fileName = fileName; + var asmName = new AssemblyName(Path.GetFileName(fileName)); + asmName.Version = new Version(VersionMajor, VersionMinor); + _emitter = new TypeSystemMetadataEmitter(asmName, context); + _emitter.AllowUseOfAddGlobalMethod(); + } + + internal override void Begin() + { + } + + protected override void DumpObjectNode(NameMangler mangler, ObjectNode node, ObjectData objectData) + { + string mangledName = null; + if (node is ISymbolNode symbol) + { + _utf8StringBuilder.Clear(); + symbol.AppendMangledName(mangler, _utf8StringBuilder); + mangledName = _utf8StringBuilder.ToString(); + } + + switch (node) + { + case EETypeNode eeType: + SerializeSimpleEntry(_types, eeType.Type, mangledName, objectData); + break; + case IMethodBodyNode methodBody: + var codeInfo = (INodeWithCodeInfo)node; + _methods.Add(methodBody.Method, (mangledName, objectData.Data.Length, codeInfo.GCInfo.Length)); + break; + case MethodExceptionHandlingInfoNode ehInfoNode: + _methodEhInfo.Add(ehInfoNode.Method, objectData.Data.Length); + break; + default: + string nodeName = GetObjectNodeName(node); + if (!_blobs.TryGetValue(nodeName, out int size)) + size = 0; + _blobs[nodeName] = size + objectData.Data.Length; + break; + } + } + + private void SerializeSimpleEntry(InstructionEncoder encoder, TypeSystemEntity entity, string mangledName, ObjectData blob) + { + encoder.OpCode(ILOpCode.Ldtoken); + encoder.Token(_emitter.EmitMetadataHandleForTypeSystemEntity(entity)); + encoder.LoadString(_emitter.GetUserStringHandle(mangledName)); + encoder.LoadConstantI4(blob.Data.Length); + } + + internal override void End() + { + var methods = new InstructionEncoder(new BlobBuilder()); + foreach (var m in _methods) + { + methods.OpCode(ILOpCode.Ldtoken); + methods.Token(_emitter.EmitMetadataHandleForTypeSystemEntity(m.Key)); + methods.LoadString(_emitter.GetUserStringHandle(m.Value.MangledName)); + methods.LoadConstantI4(m.Value.Size); + methods.LoadConstantI4(m.Value.GcInfoSize); + methods.LoadConstantI4(_methodEhInfo.GetValueOrDefault(m.Key)); + } + + var blobs = new InstructionEncoder(new BlobBuilder()); + foreach (var b in _blobs) + { + blobs.LoadString(_emitter.GetUserStringHandle(b.Key)); + blobs.LoadConstantI4(b.Value); + } + + _emitter.AddGlobalMethod("Methods", methods, 0); + _emitter.AddGlobalMethod("Types", _types, 0); + _emitter.AddGlobalMethod("Blobs", blobs, 0); + + using (var fs = File.OpenWrite(_fileName)) + _emitter.SerializeToStream(fs); + } + } +} diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ObjectDumper.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ObjectDumper.cs index 8d56104eb2502b..10d229da90c47e 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ObjectDumper.cs +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/ObjectDumper.cs @@ -2,9 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.IO; -using System.Security.Cryptography; -using System.Xml; +using System.Collections.Generic; using Internal.Text; @@ -14,31 +12,14 @@ namespace ILCompiler { - public class ObjectDumper : IObjectDumper + public abstract class ObjectDumper : IObjectDumper { - private readonly string _fileName; - private SHA256 _sha256; - private XmlWriter _writer; + internal abstract void Begin(); + internal abstract void End(); + void IObjectDumper.DumpObjectNode(NameMangler mangler, ObjectNode node, ObjectData objectData) => DumpObjectNode(mangler, node, objectData); + protected abstract void DumpObjectNode(NameMangler mangler, ObjectNode node, ObjectData objectData); - public ObjectDumper(string fileName) - { - _fileName = fileName; - } - - internal void Begin() - { - var settings = new XmlWriterSettings - { - CloseOutput = true, - Indent = true, - }; - - _sha256 = SHA256.Create(); - _writer = XmlWriter.Create(File.CreateText(_fileName), settings); - _writer.WriteStartElement("ObjectNodes"); - } - - private static string GetObjectNodeName(ObjectNode node) + protected static string GetObjectNodeName(ObjectNode node) { string name = node.GetType().Name; @@ -54,46 +35,42 @@ private static string GetObjectNodeName(ObjectNode node) return name; } - void IObjectDumper.DumpObjectNode(NameMangler mangler, ObjectNode node, ObjectData objectData) + public static ObjectDumper Compose(IEnumerable dumpers) { - string name = null; - - _writer.WriteStartElement(GetObjectNodeName(node)); - - var symbolNode = node as ISymbolNode; - if (symbolNode != null) - { - Utf8StringBuilder sb = new Utf8StringBuilder(); - symbolNode.AppendMangledName(mangler, sb); - name = sb.ToString(); - _writer.WriteAttributeString("Name", name); - } + var dumpersList = new ArrayBuilder(); - _writer.WriteAttributeString("Length", objectData.Data.Length.ToStringInvariant()); - _writer.WriteAttributeString("Hash", HashData(objectData.Data)); - _writer.WriteEndElement(); + foreach (var dumper in dumpers) + dumpersList.Add(dumper); - var nodeWithCodeInfo = node as INodeWithCodeInfo; - if (nodeWithCodeInfo != null) + return dumpersList.Count switch { - _writer.WriteStartElement("GCInfo"); - _writer.WriteAttributeString("Name", name); - _writer.WriteAttributeString("Length", nodeWithCodeInfo.GCInfo.Length.ToStringInvariant()); - _writer.WriteAttributeString("Hash", HashData(nodeWithCodeInfo.GCInfo)); - _writer.WriteEndElement(); - } + 0 => null, + 1 => dumpersList[0], + _ => new ComposedObjectDumper(dumpersList.ToArray()), + }; } - private string HashData(byte[] data) + private class ComposedObjectDumper : ObjectDumper { - return BitConverter.ToString(_sha256.ComputeHash(data)).Replace("-", "").ToLower(); - } + private readonly ObjectDumper[] _dumpers; - internal void End() - { - _writer.WriteEndElement(); - _writer.Dispose(); - _writer = null; + public ComposedObjectDumper(ObjectDumper[] dumpers) => _dumpers = dumpers; + + protected override void DumpObjectNode(NameMangler mangler, ObjectNode node, ObjectData objectData) + { + foreach (var d in _dumpers) + d.DumpObjectNode(mangler, node, objectData); + } + internal override void Begin() + { + foreach (var d in _dumpers) + d.Begin(); + } + internal override void End() + { + foreach (var d in _dumpers) + d.End(); + } } } } diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/XmlObjectDumper.cs b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/XmlObjectDumper.cs new file mode 100644 index 00000000000000..b1fa747348f296 --- /dev/null +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/Compiler/XmlObjectDumper.cs @@ -0,0 +1,83 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Security.Cryptography; +using System.Xml; + +using Internal.Text; + +using ILCompiler.DependencyAnalysis; + +using ObjectData = ILCompiler.DependencyAnalysis.ObjectNode.ObjectData; + +namespace ILCompiler +{ + public class XmlObjectDumper : ObjectDumper + { + private readonly string _fileName; + private SHA256 _sha256; + private XmlWriter _writer; + + public XmlObjectDumper(string fileName) + { + _fileName = fileName; + } + + internal override void Begin() + { + var settings = new XmlWriterSettings + { + CloseOutput = true, + Indent = true, + }; + + _sha256 = SHA256.Create(); + _writer = XmlWriter.Create(File.CreateText(_fileName), settings); + _writer.WriteStartElement("ObjectNodes"); + } + + protected override void DumpObjectNode(NameMangler mangler, ObjectNode node, ObjectData objectData) + { + string name = null; + + _writer.WriteStartElement(GetObjectNodeName(node)); + + var symbolNode = node as ISymbolNode; + if (symbolNode != null) + { + Utf8StringBuilder sb = new Utf8StringBuilder(); + symbolNode.AppendMangledName(mangler, sb); + name = sb.ToString(); + _writer.WriteAttributeString("Name", name); + } + + _writer.WriteAttributeString("Length", objectData.Data.Length.ToStringInvariant()); + _writer.WriteAttributeString("Hash", HashData(objectData.Data)); + _writer.WriteEndElement(); + + var nodeWithCodeInfo = node as INodeWithCodeInfo; + if (nodeWithCodeInfo != null) + { + _writer.WriteStartElement("GCInfo"); + _writer.WriteAttributeString("Name", name); + _writer.WriteAttributeString("Length", nodeWithCodeInfo.GCInfo.Length.ToStringInvariant()); + _writer.WriteAttributeString("Hash", HashData(nodeWithCodeInfo.GCInfo)); + _writer.WriteEndElement(); + } + } + + private string HashData(byte[] data) + { + return BitConverter.ToString(_sha256.ComputeHash(data)).Replace("-", "").ToLower(); + } + + internal override void End() + { + _writer.WriteEndElement(); + _writer.Dispose(); + _writer = null; + } + } +} diff --git a/src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj b/src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj index 2117a90d60bf76..035e1a74c9688e 100644 --- a/src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj +++ b/src/coreclr/tools/aot/ILCompiler.Compiler/ILCompiler.Compiler.csproj @@ -177,6 +177,9 @@ TypeSystem\CodeGen\NativeStructType.CodeGen.cs + + TypeSystem\MetadataEmitter\TypeSystemMetadataEmitter.cs + Common\GCDescEncoder.cs @@ -422,6 +425,7 @@ + @@ -592,6 +596,7 @@ + diff --git a/src/coreclr/tools/aot/ILCompiler/Program.cs b/src/coreclr/tools/aot/ILCompiler/Program.cs index 8e00759aaa857c..164957c23b32a7 100644 --- a/src/coreclr/tools/aot/ILCompiler/Program.cs +++ b/src/coreclr/tools/aot/ILCompiler/Program.cs @@ -53,6 +53,7 @@ internal class Program private bool _noPreinitStatics; private bool _emitStackTraceData; private string _mapFileName; + private string _mstatFileName; private string _metadataLogFileName; private bool _noMetadataBlocking; private string _reflectionData; @@ -202,6 +203,7 @@ private ArgumentSyntax ParseCommandLine(string[] args) syntax.DefineOptionList("codegenopt", ref _codegenOptions, "Define a codegen option"); syntax.DefineOptionList("rdxml", ref _rdXmlFilePaths, "RD.XML file(s) for compilation"); syntax.DefineOption("map", ref _mapFileName, "Generate a map file"); + syntax.DefineOption("mstat", ref _mstatFileName, "Generate an mstat file"); syntax.DefineOption("metadatalog", ref _metadataLogFileName, "Generate a metadata log file"); syntax.DefineOption("nometadatablocking", ref _noMetadataBlocking, "Ignore metadata blocking for internal implementation details"); syntax.DefineOption("completetypemetadata", ref _completeTypesMetadata, "Generate complete metadata for types"); @@ -925,9 +927,15 @@ void RunScanner() ICompilation compilation = builder.ToCompilation(); - ObjectDumper dumper = _mapFileName != null ? new ObjectDumper(_mapFileName) : null; + List dumpers = new List(); - CompilationResults compilationResults = compilation.Compile(_outputFilePath, dumper); + if (_mapFileName != null) + dumpers.Add(new XmlObjectDumper(_mapFileName)); + + if (_mstatFileName != null) + dumpers.Add(new MstatObjectDumper(_mstatFileName, typeSystemContext)); + + CompilationResults compilationResults = compilation.Compile(_outputFilePath, ObjectDumper.Compose(dumpers)); if (_exportsFile != null) { ExportsFileWriter defFileWriter = new ExportsFileWriter(typeSystemContext, _exportsFile);