diff --git a/eng/Versions.props b/eng/Versions.props index dfab66e55a9..1fa6e0bb15e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -103,6 +103,7 @@ 4.11.1 4.3.0 4.3.0 + 4.7.1 3.8.0-5.20570.14 $(RoslynVersion) diff --git a/src/fsharp/CompilerImports.fs b/src/fsharp/CompilerImports.fs index c730190ad0f..82b55aaff7e 100644 --- a/src/fsharp/CompilerImports.fs +++ b/src/fsharp/CompilerImports.fs @@ -90,9 +90,11 @@ let PickleToResource inMem file (g: TcGlobals) scope rName p x = let bytes = pickleObjWithDanglingCcus inMem file g scope p x let byteStorage = if inMem then - ByteStorage.FromByteArrayAndCopy(bytes, useBackingMemoryMappedFile = true) + ByteStorage.FromMemoryAndCopy(bytes.AsMemory(), useBackingMemoryMappedFile = true) else - ByteStorage.FromByteArray(bytes) + ByteStorage.FromByteArray(bytes.AsMemory().ToArray()) + + (bytes :> IDisposable).Dispose() { Name = rName Location = ILResourceLocation.Local(byteStorage) diff --git a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj index cfbae17b9a1..dd96b7da59e 100644 --- a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj +++ b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj @@ -59,6 +59,7 @@ + @@ -987,6 +988,7 @@ + diff --git a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec index c7f35f65030..065f1cdb31c 100644 --- a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec +++ b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec @@ -30,6 +30,7 @@ + diff --git a/src/fsharp/IlxGen.fs b/src/fsharp/IlxGen.fs index 6a180180c7f..6cefacd50c3 100644 --- a/src/fsharp/IlxGen.fs +++ b/src/fsharp/IlxGen.fs @@ -2027,9 +2027,9 @@ let GenString cenv cgbuf s = let GenConstArray cenv (cgbuf: CodeGenBuffer) eenv ilElementType (data:'a[]) (write: ByteBuffer -> 'a -> unit) = let g = cenv.g - let buf = ByteBuffer.Create data.Length + use buf = ByteBuffer.Create data.Length data |> Array.iter (write buf) - let bytes = buf.Close() + let bytes = buf.AsMemory().ToArray() let ilArrayType = mkILArr1DTy ilElementType if data.Length = 0 then CG.EmitInstrs cgbuf (pop 0) (Push [ilArrayType]) [ mkLdcInt32 0; I_newarr (ILArrayShape.SingleDimensional, ilElementType); ] diff --git a/src/fsharp/QuotationPickler.fs b/src/fsharp/QuotationPickler.fs index 6a7f3689b7b..abcf1a42882 100644 --- a/src/fsharp/QuotationPickler.fs +++ b/src/fsharp/QuotationPickler.fs @@ -2,6 +2,7 @@ module internal FSharp.Compiler.QuotationPickler +open System open System.Text open FSharp.Compiler.IO open Internal.Utilities @@ -245,6 +246,10 @@ let SerializedReflectedDefinitionsResourceNameBase = "ReflectedDefinitions" let freshVar (n, ty, mut) = { vText=n; vType=ty; vMutable=mut } +/// Arbitrary value +[] +let PickleBufferCapacity = 100000 + module SimplePickle = type Table<'T> = @@ -311,6 +316,11 @@ module SimplePickle = p_int32 (len) st st.os.EmitBytes s + let p_memory (s:ReadOnlyMemory) st = + let len = s.Length + p_int32 (len) st + st.os.EmitMemory s + let prim_pstring (s:string) st = let bytes = Encoding.UTF8.GetBytes s let len = bytes.Length @@ -363,20 +373,26 @@ module SimplePickle = | h :: t -> p_byte 1 st; f h st; p_list f t st let pickle_obj p x = + let st1 = + { os = ByteBuffer.Create(PickleBufferCapacity, useArrayPool = true) + ostrings=Table<_>.Create() } let stringTab, phase1bytes = - let st1 = - { os = ByteBuffer.Create 100000 - ostrings=Table<_>.Create() } p x st1 - st1.ostrings.AsList, st1.os.Close() + st1.ostrings.AsList, st1.os.AsMemory() + let phase2data = (stringTab, phase1bytes) + + let st2 = + { os = ByteBuffer.Create(PickleBufferCapacity, useArrayPool = true) + ostrings=Table<_>.Create() } let phase2bytes = - let st2 = - { os = ByteBuffer.Create 100000 - ostrings=Table<_>.Create() } - p_tup2 (p_list prim_pstring) p_bytes phase2data st2 - st2.os.Close() - phase2bytes + p_tup2 (p_list prim_pstring) p_memory phase2data st2 + st2.os.AsMemory() + + let finalBytes = phase2bytes.ToArray() + (st1.os :> IDisposable).Dispose() + (st2.os :> IDisposable).Dispose() + finalBytes open SimplePickle diff --git a/src/fsharp/TypedTree.fs b/src/fsharp/TypedTree.fs index 5693220abe3..aa1ae4ee748 100644 --- a/src/fsharp/TypedTree.fs +++ b/src/fsharp/TypedTree.fs @@ -4637,7 +4637,7 @@ type TOp = | Array /// Constant byte arrays (used for parser tables and other embedded data) - | Bytes of byte[] + | Bytes of byte[] /// Constant uint16 arrays (used for parser tables) | UInt16s of uint16[] diff --git a/src/fsharp/TypedTreePickle.fs b/src/fsharp/TypedTreePickle.fs index 9e48b1c23fb..b8325c261d1 100644 --- a/src/fsharp/TypedTreePickle.fs +++ b/src/fsharp/TypedTreePickle.fs @@ -213,6 +213,11 @@ let p_bytes (s: byte[]) st = p_int32 len st st.os.EmitBytes s +let p_memory (s: System.ReadOnlyMemory) st = + let len = s.Length + p_int32 len st + st.os.EmitMemory s + let p_prim_string (s: string) st = let bytes = Encoding.UTF8.GetBytes s let len = bytes.Length @@ -775,10 +780,13 @@ let p_encoded_simpletyp x st = p_int x st let p_encoded_anoninfo x st = p_int x st let p_simpletyp x st = p_int (encode_simpletyp st.occus st.ostrings st.onlerefs st.osimpletys st.oscope x) st +/// Arbitrary value +[] +let PickleBufferCapacity = 100000 + let pickleObjWithDanglingCcus inMem file g scope p x = - let ccuNameTab, (ntycons, ntypars, nvals, nanoninfos), stringTab, pubpathTab, nlerefTab, simpleTyTab, phase1bytes = - let st1 = - { os = ByteBuffer.Create 100000 + let st1 = + { os = ByteBuffer.Create(PickleBufferCapacity, useArrayPool = true) oscope=scope occus= Table<_>.Create "occus" oentities=NodeOutTable<_, _>.Create((fun (tc: Tycon) -> tc.Stamp), (fun tc -> tc.LogicalName), (fun tc -> tc.Range), (fun osgn -> osgn), "otycons") @@ -793,31 +801,32 @@ let pickleObjWithDanglingCcus inMem file g scope p x = ofile=file oInMem=inMem isStructThisArgPos = false} + let ccuNameTab, (ntycons, ntypars, nvals, nanoninfos), stringTab, pubpathTab, nlerefTab, simpleTyTab, phase1bytes = p x st1 let sizes = st1.oentities.Size, st1.otypars.Size, st1.ovals.Size, st1.oanoninfos.Size - st1.occus, sizes, st1.ostrings, st1.opubpaths, st1.onlerefs, st1.osimpletys, st1.os.Close() - + st1.occus, sizes, st1.ostrings, st1.opubpaths, st1.onlerefs, st1.osimpletys, st1.os.AsMemory() + + let st2 = + { os = ByteBuffer.Create(PickleBufferCapacity, useArrayPool = true) + oscope=scope + occus= Table<_>.Create "occus (fake)" + oentities=NodeOutTable<_, _>.Create((fun (tc: Tycon) -> tc.Stamp), (fun tc -> tc.LogicalName), (fun tc -> tc.Range), (fun osgn -> osgn), "otycons") + otypars=NodeOutTable<_, _>.Create((fun (tp: Typar) -> tp.Stamp), (fun tp -> tp.DisplayName), (fun tp -> tp.Range), (fun osgn -> osgn), "otypars") + ovals=NodeOutTable<_, _>.Create((fun (v: Val) -> v.Stamp), (fun v -> v.LogicalName), (fun v -> v.Range), (fun osgn -> osgn), "ovals") + oanoninfos=NodeOutTable<_, _>.Create((fun (v: AnonRecdTypeInfo) -> v.Stamp), (fun v -> string v.Stamp), (fun _ -> range0), id, "oanoninfos") + ostrings=Table<_>.Create "ostrings (fake)" + opubpaths=Table<_>.Create "opubpaths (fake)" + onlerefs=Table<_>.Create "onlerefs (fake)" + osimpletys=Table<_>.Create "osimpletys (fake)" + oglobals=g + ofile=file + oInMem=inMem + isStructThisArgPos = false } let phase2bytes = - let st2 = - { os = ByteBuffer.Create 100000 - oscope=scope - occus= Table<_>.Create "occus (fake)" - oentities=NodeOutTable<_, _>.Create((fun (tc: Tycon) -> tc.Stamp), (fun tc -> tc.LogicalName), (fun tc -> tc.Range), (fun osgn -> osgn), "otycons") - otypars=NodeOutTable<_, _>.Create((fun (tp: Typar) -> tp.Stamp), (fun tp -> tp.DisplayName), (fun tp -> tp.Range), (fun osgn -> osgn), "otypars") - ovals=NodeOutTable<_, _>.Create((fun (v: Val) -> v.Stamp), (fun v -> v.LogicalName), (fun v -> v.Range), (fun osgn -> osgn), "ovals") - oanoninfos=NodeOutTable<_, _>.Create((fun (v: AnonRecdTypeInfo) -> v.Stamp), (fun v -> string v.Stamp), (fun _ -> range0), id, "oanoninfos") - ostrings=Table<_>.Create "ostrings (fake)" - opubpaths=Table<_>.Create "opubpaths (fake)" - onlerefs=Table<_>.Create "onlerefs (fake)" - osimpletys=Table<_>.Create "osimpletys (fake)" - oglobals=g - ofile=file - oInMem=inMem - isStructThisArgPos = false } p_array p_encoded_ccuref ccuNameTab.AsArray st2 // Add a 4th integer indicated by a negative 1st integer let z1 = if nanoninfos > 0 then -ntycons-1 else ntycons @@ -830,11 +839,14 @@ let pickleObjWithDanglingCcus inMem file g scope p x = (p_array p_encoded_pubpath) (p_array p_encoded_nleref) (p_array p_encoded_simpletyp) - p_bytes + p_memory (stringTab.AsArray, pubpathTab.AsArray, nlerefTab.AsArray, simpleTyTab.AsArray, phase1bytes) st2 - st2.os.Close() - phase2bytes + st2.os + + let finalBytes = phase2bytes + (st1.os :> System.IDisposable).Dispose() + finalBytes let check (ilscope: ILScopeRef) (inMap : NodeInTable<_, _>) = for i = 0 to inMap.Count - 1 do diff --git a/src/fsharp/TypedTreePickle.fsi b/src/fsharp/TypedTreePickle.fsi index 7a81358a003..5a2ade30289 100644 --- a/src/fsharp/TypedTreePickle.fsi +++ b/src/fsharp/TypedTreePickle.fsi @@ -84,7 +84,7 @@ val internal p_ty : pickler val internal pickleCcuInfo : pickler /// Serialize an arbitrary object using the given pickler -val pickleObjWithDanglingCcus : inMem: bool -> file: string -> TcGlobals -> scope:CcuThunk -> pickler<'T> -> 'T -> byte[] +val pickleObjWithDanglingCcus : inMem: bool -> file: string -> TcGlobals -> scope:CcuThunk -> pickler<'T> -> 'T -> ByteBuffer /// The type of state unpicklers read from type ReaderState diff --git a/src/fsharp/absil/ilsupp.fs b/src/fsharp/absil/ilsupp.fs index fa77befa8d3..cf2658bd31a 100644 --- a/src/fsharp/absil/ilsupp.fs +++ b/src/fsharp/absil/ilsupp.fs @@ -91,7 +91,7 @@ type IMAGE_FILE_HEADER (m: int16, secs: int16, tds: int32, ptst: int32, nos: int with get() = 20 member x.toBytes () = - let buf = ByteBuffer.Create IMAGE_FILE_HEADER.Width + use buf = ByteBuffer.Create IMAGE_FILE_HEADER.Width buf.EmitUInt16 (uint16 machine) buf.EmitUInt16 (uint16 numberOfSections) buf.EmitInt32 timeDateStamp @@ -99,7 +99,7 @@ type IMAGE_FILE_HEADER (m: int16, secs: int16, tds: int32, ptst: int32, nos: int buf.EmitInt32 numberOfSymbols buf.EmitUInt16 (uint16 sizeOfOptionalHeader) buf.EmitUInt16 (uint16 characteristics) - buf.Close() + buf.AsMemory().ToArray() let bytesToIFH (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_FILE_HEADER.Width then @@ -172,7 +172,7 @@ type IMAGE_SECTION_HEADER(n: int64, ai: int32, va: int32, srd: int32, prd: int32 with get() = 40 member x.toBytes () = - let buf = ByteBuffer.Create IMAGE_SECTION_HEADER.Width + use buf = ByteBuffer.Create IMAGE_SECTION_HEADER.Width buf.EmitInt64 name buf.EmitInt32 addressInfo buf.EmitInt32 virtualAddress @@ -183,7 +183,7 @@ type IMAGE_SECTION_HEADER(n: int64, ai: int32, va: int32, srd: int32, prd: int32 buf.EmitUInt16 (uint16 numberOfRelocations) buf.EmitUInt16 (uint16 numberOfLineNumbers) buf.EmitInt32 characteristics - buf.Close() + buf.AsMemory().ToArray() let bytesToISH (buffer: byte[]) (offset: int) = @@ -236,14 +236,14 @@ type IMAGE_SYMBOL(n: int64, v: int32, sn: int16, t: int16, sc: byte, nas: byte) with get() = 18 member x.toBytes() = - let buf = ByteBuffer.Create IMAGE_SYMBOL.Width + use buf = ByteBuffer.Create IMAGE_SYMBOL.Width buf.EmitInt64 name buf.EmitInt32 value buf.EmitUInt16 (uint16 sectionNumber) buf.EmitUInt16 (uint16 stype) buf.EmitByte storageClass buf.EmitByte numberOfAuxSymbols - buf.Close() + buf.AsMemory().ToArray() let bytesToIS (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_SYMBOL.Width then @@ -280,11 +280,11 @@ type IMAGE_RELOCATION(va: int32, sti: int32, t: int16) = with get() = 10 member x.toBytes() = - let buf = ByteBuffer.Create IMAGE_RELOCATION.Width + use buf = ByteBuffer.Create IMAGE_RELOCATION.Width buf.EmitInt32 virtualAddress buf.EmitInt32 symbolTableIndex buf.EmitUInt16 (uint16 ty) - buf.Close() + buf.AsMemory().ToArray() let bytesToIR (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_RELOCATION.Width then @@ -328,14 +328,14 @@ type IMAGE_RESOURCE_DIRECTORY(c: int32, tds: int32, mjv: int16, mnv: int16, nne: static member Width = 16 member x.toBytes () = - let buf = ByteBuffer.Create IMAGE_RESOURCE_DIRECTORY.Width + use buf = ByteBuffer.Create IMAGE_RESOURCE_DIRECTORY.Width buf.EmitInt32 characteristics buf.EmitInt32 timeDateStamp buf.EmitUInt16 (uint16 majorVersion) buf.EmitUInt16 (uint16 minorVersion) buf.EmitUInt16 (uint16 numberOfNamedEntries) buf.EmitUInt16 (uint16 numberOfIdEntries) - buf.Close() + buf.AsMemory().ToArray() let bytesToIRD (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_RESOURCE_DIRECTORY.Width then @@ -368,10 +368,10 @@ type IMAGE_RESOURCE_DIRECTORY_ENTRY(n: int32, o: int32) = static member Width = 8 member x.toBytes () = - let buf = ByteBuffer.Create IMAGE_RESOURCE_DIRECTORY_ENTRY.Width + use buf = ByteBuffer.Create IMAGE_RESOURCE_DIRECTORY_ENTRY.Width buf.EmitInt32 name buf.EmitInt32 offset - buf.Close() + buf.AsMemory().ToArray() let bytesToIRDE (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_RESOURCE_DIRECTORY_ENTRY.Width then @@ -401,7 +401,7 @@ type IMAGE_RESOURCE_DATA_ENTRY(o: int32, s: int32, c: int32, r: int32) = static member Width = 16 member x.toBytes() = - let buf = ByteBuffer.Create IMAGE_RESOURCE_DATA_ENTRY.Width + use buf = ByteBuffer.Create IMAGE_RESOURCE_DATA_ENTRY.Width buf.EmitInt32 offsetToData buf.EmitInt32 size buf.EmitInt32 codePage @@ -466,7 +466,7 @@ type ResFormatHeader() = static member Width = 32 member x.toBytes() = - let buf = ByteBuffer.Create ResFormatHeader.Width + use buf = ByteBuffer.Create ResFormatHeader.Width buf.EmitInt32 dwDataSize buf.EmitInt32 dwHeaderSize buf.EmitInt32 dwTypeID @@ -476,7 +476,7 @@ type ResFormatHeader() = buf.EmitUInt16 (uint16 wLangID) buf.EmitInt32 dwVersion buf.EmitInt32 dwCharacteristics - buf.Close() + buf.AsMemory().ToArray() type ResFormatNode(tid: int32, nid: int32, lid: int32, dataOffset: int32, pbLinkedResource: byte[]) = let mutable resHdr = ResFormatHeader() diff --git a/src/fsharp/absil/ilwrite.fs b/src/fsharp/absil/ilwrite.fs index 6bb374edad4..2abd677d08e 100644 --- a/src/fsharp/absil/ilwrite.fs +++ b/src/fsharp/absil/ilwrite.fs @@ -2,6 +2,7 @@ module internal FSharp.Compiler.AbstractIL.ILBinaryWriter +open System open System.Collections.Generic open System.IO @@ -47,7 +48,10 @@ let dw0 n = byte (n &&& 0xFFL) let bitsOfSingle (x: float32) = System.BitConverter.ToInt32(System.BitConverter.GetBytes x, 0) let bitsOfDouble (x: float) = System.BitConverter.DoubleToInt64Bits x -let emitBytesViaBuffer f = let bb = ByteBuffer.Create 10 in f bb; bb.Close() +/// Arbitrary value +[] +let EmitBytesViaBufferCapacity = 10 +let emitBytesViaBuffer f = use bb = ByteBuffer.Create EmitBytesViaBufferCapacity in f bb; bb.AsMemory().ToArray() /// Alignment and padding let align alignment n = ((n + alignment - 1) / alignment) * alignment @@ -543,10 +547,16 @@ type cenv = cenv.codeChunks.EmitBytes code cenv.nextCodeAddr <- cenv.nextCodeAddr + code.Length - member cenv.GetCode() = cenv.codeChunks.Close() + member cenv.GetCode() = cenv.codeChunks.AsMemory().ToArray() override x.ToString() = "" + interface IDisposable with + member this.Dispose() = + (this.codeChunks :> IDisposable).Dispose() + (this.data :> IDisposable).Dispose() + (this.resources :> IDisposable).Dispose() + let FindOrAddSharedRow (cenv: cenv) tbl x = cenv.GetTable(tbl).FindOrAddSharedEntry x // Shared rows must be hash-cons'd to be made unique (no duplicates according to contents) @@ -1446,6 +1456,10 @@ type ExceptionClauseKind = type ExceptionClauseSpec = (int * int * int * int * ExceptionClauseKind) +/// Arbitrary value +[] +let CodeBufferCapacity = 200 + type CodeBuffer = // -------------------------------------------------------------------- @@ -1465,9 +1479,13 @@ type CodeBuffer = mutable seh: ExceptionClauseSpec list seqpoints: ResizeArray } + interface IDisposable with + member this.Dispose() = + (this.code :> IDisposable).Dispose() + static member Create _nm = { seh = [] - code= ByteBuffer.Create 200 + code= ByteBuffer.Create CodeBufferCapacity reqdBrFixups=[] reqdStringFixupsInMethod=[] availBrFixups = Dictionary<_, _>(10, HashIdentity.Structural) @@ -1534,7 +1552,7 @@ module Codebuf = let applyBrFixups (origCode : byte[]) origExnClauses origReqdStringFixups (origAvailBrFixups: Dictionary) origReqdBrFixups origSeqPoints origScopes = let orderedOrigReqdBrFixups = origReqdBrFixups |> List.sortBy (fun (_, fixupLoc, _) -> fixupLoc) - let newCode = ByteBuffer.Create origCode.Length + use newCode = ByteBuffer.Create origCode.Length // Copy over all the code, working out whether the branches will be short // or long and adjusting the branch destinations. Record an adjust function to adjust all the other @@ -1639,7 +1657,7 @@ module Codebuf = let (origStartOfNoBranchBlock, _, newStartOfNoBranchBlock) = arr.[i] addr - (origStartOfNoBranchBlock - newStartOfNoBranchBlock) - newCode.Close(), + newCode.AsMemory().ToArray(), !newReqdBrFixups, adjuster @@ -2138,9 +2156,9 @@ module Codebuf = localsTree let EmitTopCode cenv localSigs env nm code = - let codebuf = CodeBuffer.Create nm + use codebuf = CodeBuffer.Create nm let origScopes = emitCode cenv localSigs codebuf env code - let origCode = codebuf.code.Close() + let origCode = codebuf.code.AsMemory().ToArray() let origExnClauses = List.rev codebuf.seh let origReqdStringFixups = codebuf.reqdStringFixupsInMethod let origAvailBrFixups = codebuf.availBrFixups @@ -2179,7 +2197,7 @@ let GenILMethodBody mname cenv env (il: ILMethodBody) = let requiredStringFixups, seh, code, seqpoints, scopes = Codebuf.EmitTopCode cenv localSigs env mname il.Code let codeSize = code.Length - let methbuf = ByteBuffer.Create (codeSize * 3) + use methbuf = ByteBuffer.Create (codeSize * 3) // Do we use the tiny format? if isNil il.Locals && il.MaxStack <= 8 && isNil seh && codeSize < 64 then // Use Tiny format @@ -2189,7 +2207,7 @@ let GenILMethodBody mname cenv env (il: ILMethodBody) = methbuf.EmitByte (byte codeSize <<< 2 ||| e_CorILMethod_TinyFormat) methbuf.EmitBytes code methbuf.EmitPadding codePadding - 0x0, (requiredStringFixups', methbuf.Close()), seqpoints, scopes + 0x0, (requiredStringFixups', methbuf.AsMemory().ToArray()), seqpoints, scopes else // Use Fat format let flags = @@ -2263,7 +2281,7 @@ let GenILMethodBody mname cenv env (il: ILMethodBody) = let requiredStringFixups' = (12, requiredStringFixups) - localToken, (requiredStringFixups', methbuf.Close()), seqpoints, scopes + localToken, (requiredStringFixups', methbuf.AsMemory().ToArray()), seqpoints, scopes // -------------------------------------------------------------------- // ILFieldDef --> FieldDef Row @@ -2859,10 +2877,20 @@ let GenModule (cenv : cenv) (modul: ILModuleDef) = GenTypeDefsPass4 [] cenv tds reportTime cenv.showTimes "Module Generation Pass 4" +/// Arbitrary value +[] +let CodeChunkCapacity = 40000 +/// Arbitrary value +[] +let DataCapacity = 200 +/// Arbitrary value +[] +let ResourceCapacity = 200 + let generateIL requiredDataFixups (desiredMetadataVersion, generatePdb, ilg : ILGlobals, emitTailcalls, deterministic, showTimes) (m : ILModuleDef) cilStartAddress normalizeAssemblyRefs = let isDll = m.IsDLL - let cenv = + use cenv = { emitTailcalls=emitTailcalls deterministic = deterministic showTimes=showTimes @@ -2870,10 +2898,10 @@ let generateIL requiredDataFixups (desiredMetadataVersion, generatePdb, ilg : IL desiredMetadataVersion=desiredMetadataVersion requiredDataFixups= requiredDataFixups requiredStringFixups = [] - codeChunks=ByteBuffer.Create 40000 + codeChunks=ByteBuffer.Create(CodeChunkCapacity, useArrayPool = true) nextCodeAddr = cilStartAddress - data = ByteBuffer.Create 200 - resources = ByteBuffer.Create 200 + data = ByteBuffer.Create DataCapacity + resources = ByteBuffer.Create ResourceCapacity tables= Array.init 64 (fun i -> if (i = TableNames.AssemblyRef.Index || @@ -2959,8 +2987,8 @@ let generateIL requiredDataFixups (desiredMetadataVersion, generatePdb, ilg : IL getUncodedToken TableNames.Event (cenv.eventDefs.GetTableEntry (EventKey (tidx, ed.Name)))) } reportTime cenv.showTimes "Finalize Module Generation Results" // New return the results - let data = cenv.data.Close() - let resources = cenv.resources.Close() + let data = cenv.data.AsMemory().ToArray() + let resources = cenv.resources.AsMemory().ToArray() (strings, userStrings, blobs, guids, tables, entryPointToken, code, cenv.requiredStringFixups, data, resources, pdbData, mappings) @@ -3003,6 +3031,13 @@ module FileSystemUtilities = #endif () +/// Arbitrary value +[] +let TableCapacity = 20000 +/// Arbitrary value +[] +let MetadataCapacity = 500000 + let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailcalls, deterministic, showTimes) modul cilStartAddress normalizeAssemblyRefs = // When we know the real RVAs of the data section we fixup the references for the FieldRVA table. @@ -3230,7 +3265,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca codedBigness 2 TableNames.AssemblyRef || codedBigness 2 TableNames.TypeRef - let tablesBuf = ByteBuffer.Create 20000 + use tablesBuf = ByteBuffer.Create(TableCapacity, useArrayPool = true) // Now the coded tables themselves - first the schemata header tablesBuf.EmitIntsAsBytes @@ -3287,7 +3322,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca | _ when t <= RowElementTags.ResolutionScopeMax -> tablesBuf.EmitZTaggedIndex (t - RowElementTags.ResolutionScopeMin) 2 rsBigness n | _ -> failwith "invalid tag in row element" - tablesBuf.Close() + tablesBuf.AsMemory().ToArray() reportTime showTimes "Write Tables to tablebuf" @@ -3309,7 +3344,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca reportTime showTimes "Layout Metadata" let metadata, guidStart = - let mdbuf = ByteBuffer.Create 500000 + use mdbuf = ByteBuffer.Create(MetadataCapacity, useArrayPool = true) mdbuf.EmitIntsAsBytes [| 0x42; 0x53; 0x4a; 0x42 // Magic signature 0x01; 0x00 // Major version @@ -3378,7 +3413,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca mdbuf.EmitIntAsByte 0x00 reportTime showTimes "Write Blob Stream" // Done - close the buffer and return the result. - mdbuf.Close(), guidStart + mdbuf.AsMemory().ToArray(), guidStart // Now we know the user string tables etc. we can fixup the diff --git a/src/fsharp/lex.fsl b/src/fsharp/lex.fsl index 0aa3bc11ce3..40da3928359 100644 --- a/src/fsharp/lex.fsl +++ b/src/fsharp/lex.fsl @@ -130,8 +130,12 @@ let checkExprOp (lexbuf:UnicodeLexing.Lexbuf) = let unexpectedChar lexbuf = LEX_FAILURE (FSComp.SR.lexUnexpectedChar(lexeme lexbuf)) +/// Arbitrary value +[] +let StringCapacity = 100 + let startString args (lexbuf: UnicodeLexing.Lexbuf) = - let buf = ByteBuffer.Create 100 + let buf = ByteBuffer.Create StringCapacity let m = lexbuf.LexemeRange let startp = lexbuf.StartPos let fin = @@ -1584,7 +1588,7 @@ and tripleQuoteStringInComment n m args skip = parse and mlOnly m args skip = parse | "\"" - { let buf = ByteBuffer.Create 100 + { let buf = ByteBuffer.Create StringCapacity let m2 = lexbuf.LexemeRange let _ = singleQuoteString (buf, LexerStringFinisher.Default, m2, LexerStringKind.String, args) skip lexbuf if not skip then COMMENT (LexCont.MLOnly(args.ifdefStack, args.stringNest, m)) diff --git a/src/fsharp/lexhelp.fs b/src/fsharp/lexhelp.fs index 8fa659de6ee..7de63e244a1 100644 --- a/src/fsharp/lexhelp.fs +++ b/src/fsharp/lexhelp.fs @@ -101,12 +101,12 @@ let usingLexbufForParsing (lexbuf:Lexbuf, filename) f = //----------------------------------------------------------------------- let stringBufferAsString (buf: ByteBuffer) = - let buf = buf.Close() + let buf = buf.AsMemory() if buf.Length % 2 <> 0 then failwith "Expected even number of bytes" let chars : char[] = Array.zeroCreate (buf.Length/2) for i = 0 to (buf.Length/2) - 1 do - let hi = buf.[i*2+1] - let lo = buf.[i*2] + let hi = buf.Span.[i*2+1] + let lo = buf.Span.[i*2] let c = char (((int hi) * 256) + (int lo)) chars.[i] <- c System.String(chars) @@ -117,8 +117,8 @@ let stringBufferAsString (buf: ByteBuffer) = /// we just take every second byte we stored. Note all bytes > 127 should have been /// stored using addIntChar let stringBufferAsBytes (buf: ByteBuffer) = - let bytes = buf.Close() - Array.init (bytes.Length / 2) (fun i -> bytes.[i*2]) + let bytes = buf.AsMemory() + Array.init (bytes.Length / 2) (fun i -> bytes.Span.[i*2]) [] type LexerStringFinisherContext = @@ -185,10 +185,10 @@ let addByteChar buf (c:char) = addIntChar buf (int32 c % 256) /// Sanity check that high bytes are zeros. Further check each low byte <= 127 let stringBufferIsBytes (buf: ByteBuffer) = - let bytes = buf.Close() + let bytes = buf.AsMemory() let mutable ok = true for i = 0 to bytes.Length / 2-1 do - if bytes.[i*2+1] <> 0uy then ok <- false + if bytes.Span.[i*2+1] <> 0uy then ok <- false ok let newline (lexbuf:LexBuffer<_>) = diff --git a/src/fsharp/service/ServiceLexing.fs b/src/fsharp/service/ServiceLexing.fs index f84b53343fc..b0b66a6f35d 100644 --- a/src/fsharp/service/ServiceLexing.fs +++ b/src/fsharp/service/ServiceLexing.fs @@ -706,7 +706,7 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, | LexCont.String (ifdefs, stringNest, style, kind, m) -> lexargs.ifdefStack <- ifdefs lexargs.stringNest <- stringNest - let buf = ByteBuffer.Create 100 + use buf = ByteBuffer.Create (Lexer.StringCapacity) let args = (buf, LexerStringFinisher.Default, m, kind, lexargs) match style with | LexerStringStyle.SingleQuote -> Lexer.singleQuoteString args skip lexbuf diff --git a/src/fsharp/utils/FileSystem.fs b/src/fsharp/utils/FileSystem.fs index 2fae0ad8766..9346e6ab1a8 100644 --- a/src/fsharp/utils/FileSystem.fs +++ b/src/fsharp/utils/FileSystem.fs @@ -3,7 +3,9 @@ namespace FSharp.Compiler.IO open System open System.IO open System.IO.MemoryMappedFiles +open System.Buffers open System.Reflection +open System.Threading open System.Runtime.InteropServices open FSharp.NativeInterop open Internal.Utilities.Library @@ -317,31 +319,45 @@ type ReadOnlyByteMemory(bytes: ByteMemory) = [] module MemoryMappedFileExtensions = - type MemoryMappedFile with - static member TryFromByteMemory(bytes: ReadOnlyByteMemory) = - let length = int64 bytes.Length - if length = 0L then + + let private trymmf length copyTo = + let length = int64 length + if length = 0L then + None + else + if runningOnMono then + // mono's MemoryMappedFile implementation throws with null `mapName`, so we use byte arrays instead: https://github.com/mono/mono/issues/1024 None else - if runningOnMono then - // mono's MemoryMappedFile implementation throws with null `mapName`, so we use byte arrays instead: https://github.com/mono/mono/issues/1024 - None - else - // Try to create a memory mapped file and copy the contents of the given bytes to it. - // If this fails, then we clean up and return None. + // Try to create a memory mapped file and copy the contents of the given bytes to it. + // If this fails, then we clean up and return None. + try + let mmf = MemoryMappedFile.CreateNew(null, length, MemoryMappedFileAccess.ReadWrite, MemoryMappedFileOptions.None, HandleInheritability.None) try - let mmf = MemoryMappedFile.CreateNew(null, length, MemoryMappedFileAccess.ReadWrite, MemoryMappedFileOptions.None, HandleInheritability.None) - try - use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite) - bytes.CopyTo stream - Some mmf - with - | _ -> - mmf.Dispose() - None + use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite) + copyTo stream + Some mmf with | _ -> + mmf.Dispose() None + with + | _ -> + None + + type MemoryMappedFile with + static member TryFromByteMemory(bytes: ReadOnlyByteMemory) = + trymmf (int64 bytes.Length) bytes.CopyTo + + static member TryFromMemory(bytes: ReadOnlyMemory) = + let length = int64 bytes.Length + trymmf length + (fun stream -> + stream.SetLength(stream.Length + length) + let span = Span(stream.PositionPointer |> NativePtr.toVoidPtr, int length) + bytes.Span.CopyTo(span) + stream.Position <- stream.Position + length + ) [] module internal FileSystemUtils = @@ -755,27 +771,45 @@ type internal ByteStream = type internal ByteBuffer = - { mutable bbArray: byte[] + { useArrayPool: bool + mutable isDisposed: bool + mutable bbArray: byte[] mutable bbCurrent: int } - member buf.Ensure newSize = + member inline private buf.CheckDisposed() = + if buf.isDisposed then + raise(ObjectDisposedException(nameof(ByteBuffer))) + + member private buf.Ensure newSize = let oldBufSize = buf.bbArray.Length if newSize > oldBufSize then let old = buf.bbArray - buf.bbArray <- Bytes.zeroCreate (max newSize (oldBufSize * 2)) + buf.bbArray <- + if buf.useArrayPool then + ArrayPool.Shared.Rent (max newSize (oldBufSize * 2)) + else + Bytes.zeroCreate (max newSize (oldBufSize * 2)) Bytes.blit old 0 buf.bbArray 0 buf.bbCurrent + if buf.useArrayPool then + ArrayPool.Shared.Return old - member buf.Close () = Bytes.sub buf.bbArray 0 buf.bbCurrent + member buf.AsMemory() = + buf.CheckDisposed() + ReadOnlyMemory(buf.bbArray, 0, buf.bbCurrent) member buf.EmitIntAsByte (i:int) = + buf.CheckDisposed() let newSize = buf.bbCurrent + 1 buf.Ensure newSize buf.bbArray.[buf.bbCurrent] <- byte i buf.bbCurrent <- newSize - member buf.EmitByte (b:byte) = buf.EmitIntAsByte (int b) + member buf.EmitByte (b:byte) = + buf.CheckDisposed() + buf.EmitIntAsByte (int b) member buf.EmitIntsAsBytes (arr:int[]) = + buf.CheckDisposed() let n = arr.Length let newSize = buf.bbCurrent + n buf.Ensure newSize @@ -786,25 +820,37 @@ type internal ByteBuffer = buf.bbCurrent <- newSize member bb.FixupInt32 pos value = + bb.CheckDisposed() bb.bbArray.[pos] <- (Bytes.b0 value |> byte) bb.bbArray.[pos + 1] <- (Bytes.b1 value |> byte) bb.bbArray.[pos + 2] <- (Bytes.b2 value |> byte) bb.bbArray.[pos + 3] <- (Bytes.b3 value |> byte) member buf.EmitInt32 n = + buf.CheckDisposed() let newSize = buf.bbCurrent + 4 buf.Ensure newSize buf.FixupInt32 buf.bbCurrent n buf.bbCurrent <- newSize member buf.EmitBytes (i:byte[]) = + buf.CheckDisposed() let n = i.Length let newSize = buf.bbCurrent + n buf.Ensure newSize Bytes.blit i 0 buf.bbArray buf.bbCurrent n buf.bbCurrent <- newSize + member buf.EmitMemory (i:ReadOnlyMemory) = + buf.CheckDisposed() + let n = i.Length + let newSize = buf.bbCurrent + n + buf.Ensure newSize + i.CopyTo(Memory(buf.bbArray, buf.bbCurrent, n)) + buf.bbCurrent <- newSize + member buf.EmitByteMemory (i:ReadOnlyByteMemory) = + buf.CheckDisposed() let n = i.Length let newSize = buf.bbCurrent + n buf.Ensure newSize @@ -812,26 +858,45 @@ type internal ByteBuffer = buf.bbCurrent <- newSize member buf.EmitInt32AsUInt16 n = + buf.CheckDisposed() let newSize = buf.bbCurrent + 2 buf.Ensure newSize buf.bbArray.[buf.bbCurrent] <- (Bytes.b0 n |> byte) buf.bbArray.[buf.bbCurrent + 1] <- (Bytes.b1 n |> byte) buf.bbCurrent <- newSize - member buf.EmitBoolAsByte (b:bool) = buf.EmitIntAsByte (if b then 1 else 0) + member buf.EmitBoolAsByte (b:bool) = + buf.CheckDisposed() + buf.EmitIntAsByte (if b then 1 else 0) - member buf.EmitUInt16 (x:uint16) = buf.EmitInt32AsUInt16 (int32 x) + member buf.EmitUInt16 (x:uint16) = + buf.CheckDisposed() + buf.EmitInt32AsUInt16 (int32 x) member buf.EmitInt64 x = + buf.CheckDisposed() buf.EmitInt32 (Bytes.dWw0 x) buf.EmitInt32 (Bytes.dWw1 x) - member buf.Position = buf.bbCurrent + member buf.Position = + buf.CheckDisposed() + buf.bbCurrent - static member Create sz = - { bbArray = Bytes.zeroCreate sz + static member Create(capacity, useArrayPool) = + let useArrayPool = defaultArg useArrayPool false + { useArrayPool = useArrayPool + isDisposed = false + bbArray = if useArrayPool then ArrayPool.Shared.Rent capacity else Bytes.zeroCreate capacity bbCurrent = 0 } + interface IDisposable with + + member this.Dispose() = + if not this.isDisposed then + this.isDisposed <- true + if this.useArrayPool then + ArrayPool.Shared.Return this.bbArray + [] type ByteStorage(getByteMemory: unit -> ReadOnlyByteMemory) = @@ -868,5 +933,17 @@ type ByteStorage(getByteMemory: unit -> ReadOnlyByteMemory) = let copiedBytes = ByteMemory.FromArray(bytes.ToArray()).AsReadOnly() ByteStorage.FromByteMemory(copiedBytes) + static member FromMemoryAndCopy(bytes: ReadOnlyMemory, useBackingMemoryMappedFile: bool) = + if useBackingMemoryMappedFile then + match MemoryMappedFile.TryFromMemory(bytes) with + | Some mmf -> + ByteStorage(fun () -> ByteMemory.FromMemoryMappedFile(mmf).AsReadOnly()) + | _ -> + let copiedBytes = ByteMemory.FromArray(bytes.ToArray()).AsReadOnly() + ByteStorage.FromByteMemory(copiedBytes) + else + let copiedBytes = ByteMemory.FromArray(bytes.ToArray()).AsReadOnly() + ByteStorage.FromByteMemory(copiedBytes) + static member FromByteArrayAndCopy(bytes: byte [], useBackingMemoryMappedFile: bool) = ByteStorage.FromByteMemoryAndCopy(ByteMemory.FromArray(bytes).AsReadOnly(), useBackingMemoryMappedFile) diff --git a/src/fsharp/utils/FileSystem.fsi b/src/fsharp/utils/FileSystem.fsi index 37c2b1a66b1..14bc45a9850 100644 --- a/src/fsharp/utils/FileSystem.fsi +++ b/src/fsharp/utils/FileSystem.fsi @@ -8,6 +8,7 @@ open System.IO.MemoryMappedFiles open System.Reflection open System.Runtime.InteropServices open System.Text +open System.Runtime.CompilerServices open FSharp.NativeInterop @@ -96,6 +97,7 @@ type internal ReadOnlyByteMemory = module internal MemoryMappedFileExtensions = type MemoryMappedFile with static member TryFromByteMemory : bytes: ReadOnlyByteMemory -> MemoryMappedFile option + static member TryFromMemory : bytes: ReadOnlyMemory -> MemoryMappedFile option /// Filesystem helpers module internal FileSystemUtils = @@ -335,22 +337,52 @@ type internal ByteStream = #endif /// Imperative buffers and streams of byte[] +/// Not thread safe. [] type internal ByteBuffer = - member Close : unit -> byte[] + interface IDisposable + + [] + member AsMemory : unit -> ReadOnlyMemory + + [] member EmitIntAsByte : int -> unit + + [] member EmitIntsAsBytes : int[] -> unit + + [] member EmitByte : byte -> unit + + [] member EmitBytes : byte[] -> unit + + [] + member EmitMemory : ReadOnlyMemory -> unit + + [] member EmitByteMemory : ReadOnlyByteMemory -> unit + + [] member EmitInt32 : int32 -> unit + + [] member EmitInt64 : int64 -> unit + + [] member FixupInt32 : pos: int -> value: int32 -> unit + + [] member EmitInt32AsUInt16 : int32 -> unit + + [] member EmitBoolAsByte : bool -> unit + + [] member EmitUInt16 : uint16 -> unit + member Position : int - static member Create : int -> ByteBuffer + static member Create : capacity: int * ?useArrayPool: bool -> ByteBuffer [] type internal ByteStorage = @@ -366,5 +398,8 @@ type internal ByteStorage = /// Creates a ByteStorage that has a copy of the given ByteMemory. static member FromByteMemoryAndCopy : ReadOnlyByteMemory * useBackingMemoryMappedFile: bool -> ByteStorage + /// Creates a ByteStorage that has a copy of the given Memory. + static member FromMemoryAndCopy : ReadOnlyMemory * useBackingMemoryMappedFile: bool -> ByteStorage + /// Creates a ByteStorage that has a copy of the given byte array. static member FromByteArrayAndCopy : byte [] * useBackingMemoryMappedFile: bool -> ByteStorage diff --git a/vsintegration/Directory.Build.targets b/vsintegration/Directory.Build.targets index d47467d76c4..170ace14822 100644 --- a/vsintegration/Directory.Build.targets +++ b/vsintegration/Directory.Build.targets @@ -14,6 +14,7 @@ +