From 466db02f6e506d63bbbcb858931576f268274af2 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Wed, 9 Jun 2021 13:00:50 -0700 Subject: [PATCH 01/16] Initial work --- src/fsharp/IlxGen.fs | 4 +- src/fsharp/QuotationPickler.fs | 32 ++++++++---- src/fsharp/TypedTree.fs | 3 +- src/fsharp/absil/ilsupp.fs | 28 +++++------ src/fsharp/absil/ilwrite.fs | 2 +- src/fsharp/service/ServiceLexing.fs | 2 +- src/fsharp/utils/FileSystem.fs | 78 +++++++++++++++++++++++++---- src/fsharp/utils/FileSystem.fsi | 9 +++- 8 files changed, 117 insertions(+), 41 deletions(-) diff --git a/src/fsharp/IlxGen.fs b/src/fsharp/IlxGen.fs index 6a180180c7f..4e005dc7f29 100644 --- a/src/fsharp/IlxGen.fs +++ b/src/fsharp/IlxGen.fs @@ -2027,9 +2027,9 @@ let GenString cenv cgbuf s = let GenConstArray cenv (cgbuf: CodeGenBuffer) eenv ilElementType (data:'a[]) (write: ByteBuffer -> 'a -> unit) = let g = cenv.g - let buf = ByteBuffer.Create data.Length + use buf = ByteBuffer.Create data.Length data |> Array.iter (write buf) - let bytes = buf.Close() + let bytes = buf.GetMemory().ToArray() let ilArrayType = mkILArr1DTy ilElementType if data.Length = 0 then CG.EmitInstrs cgbuf (pop 0) (Push [ilArrayType]) [ mkLdcInt32 0; I_newarr (ILArrayShape.SingleDimensional, ilElementType); ] diff --git a/src/fsharp/QuotationPickler.fs b/src/fsharp/QuotationPickler.fs index 6a7f3689b7b..b69a16cda5a 100644 --- a/src/fsharp/QuotationPickler.fs +++ b/src/fsharp/QuotationPickler.fs @@ -2,6 +2,7 @@ module internal FSharp.Compiler.QuotationPickler +open System open System.Text open FSharp.Compiler.IO open Internal.Utilities @@ -311,6 +312,11 @@ module SimplePickle = p_int32 (len) st st.os.EmitBytes s + let p_memory (s:ReadOnlyMemory) st = + let len = s.Length + p_int32 (len) st + st.os.EmitMemory s + let prim_pstring (s:string) st = let bytes = Encoding.UTF8.GetBytes s let len = bytes.Length @@ -363,20 +369,26 @@ module SimplePickle = | h :: t -> p_byte 1 st; f h st; p_list f t st let pickle_obj p x = + let st1 = + { os = ByteBuffer.Create(100000, useArrayPool = true) + ostrings=Table<_>.Create() } let stringTab, phase1bytes = - let st1 = - { os = ByteBuffer.Create 100000 - ostrings=Table<_>.Create() } p x st1 - st1.ostrings.AsList, st1.os.Close() + st1.ostrings.AsList, st1.os.GetMemory() + let phase2data = (stringTab, phase1bytes) + + let st2 = + { os = ByteBuffer.Create(100000, useArrayPool = true) + ostrings=Table<_>.Create() } let phase2bytes = - let st2 = - { os = ByteBuffer.Create 100000 - ostrings=Table<_>.Create() } - p_tup2 (p_list prim_pstring) p_bytes phase2data st2 - st2.os.Close() - phase2bytes + p_tup2 (p_list prim_pstring) p_memory phase2data st2 + st2.os + + let finalBytes = phase2bytes.GetMemory().ToArray() + (st1.os :> IDisposable).Dispose() + (st2.os :> IDisposable).Dispose() + finalBytes open SimplePickle diff --git a/src/fsharp/TypedTree.fs b/src/fsharp/TypedTree.fs index 5693220abe3..c828f46aeb6 100644 --- a/src/fsharp/TypedTree.fs +++ b/src/fsharp/TypedTree.fs @@ -26,6 +26,7 @@ open FSharp.Compiler.SyntaxTreeOps open FSharp.Compiler.Text open FSharp.Compiler.Text.Range open FSharp.Compiler.Xml +open FSharp.Compiler.IO #if !NO_EXTENSIONTYPING open FSharp.Compiler.ExtensionTyping @@ -4637,7 +4638,7 @@ type TOp = | Array /// Constant byte arrays (used for parser tables and other embedded data) - | Bytes of byte[] + | Bytes of byte[] /// Constant uint16 arrays (used for parser tables) | UInt16s of uint16[] diff --git a/src/fsharp/absil/ilsupp.fs b/src/fsharp/absil/ilsupp.fs index fa77befa8d3..9ad0598f5c5 100644 --- a/src/fsharp/absil/ilsupp.fs +++ b/src/fsharp/absil/ilsupp.fs @@ -91,7 +91,7 @@ type IMAGE_FILE_HEADER (m: int16, secs: int16, tds: int32, ptst: int32, nos: int with get() = 20 member x.toBytes () = - let buf = ByteBuffer.Create IMAGE_FILE_HEADER.Width + use buf = ByteBuffer.Create IMAGE_FILE_HEADER.Width buf.EmitUInt16 (uint16 machine) buf.EmitUInt16 (uint16 numberOfSections) buf.EmitInt32 timeDateStamp @@ -99,7 +99,7 @@ type IMAGE_FILE_HEADER (m: int16, secs: int16, tds: int32, ptst: int32, nos: int buf.EmitInt32 numberOfSymbols buf.EmitUInt16 (uint16 sizeOfOptionalHeader) buf.EmitUInt16 (uint16 characteristics) - buf.Close() + buf.GetMemory().ToArray() let bytesToIFH (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_FILE_HEADER.Width then @@ -172,7 +172,7 @@ type IMAGE_SECTION_HEADER(n: int64, ai: int32, va: int32, srd: int32, prd: int32 with get() = 40 member x.toBytes () = - let buf = ByteBuffer.Create IMAGE_SECTION_HEADER.Width + use buf = ByteBuffer.Create IMAGE_SECTION_HEADER.Width buf.EmitInt64 name buf.EmitInt32 addressInfo buf.EmitInt32 virtualAddress @@ -183,7 +183,7 @@ type IMAGE_SECTION_HEADER(n: int64, ai: int32, va: int32, srd: int32, prd: int32 buf.EmitUInt16 (uint16 numberOfRelocations) buf.EmitUInt16 (uint16 numberOfLineNumbers) buf.EmitInt32 characteristics - buf.Close() + buf.GetMemory().ToArray() let bytesToISH (buffer: byte[]) (offset: int) = @@ -236,14 +236,14 @@ type IMAGE_SYMBOL(n: int64, v: int32, sn: int16, t: int16, sc: byte, nas: byte) with get() = 18 member x.toBytes() = - let buf = ByteBuffer.Create IMAGE_SYMBOL.Width + use buf = ByteBuffer.Create IMAGE_SYMBOL.Width buf.EmitInt64 name buf.EmitInt32 value buf.EmitUInt16 (uint16 sectionNumber) buf.EmitUInt16 (uint16 stype) buf.EmitByte storageClass buf.EmitByte numberOfAuxSymbols - buf.Close() + buf.GetMemory().ToArray() let bytesToIS (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_SYMBOL.Width then @@ -280,11 +280,11 @@ type IMAGE_RELOCATION(va: int32, sti: int32, t: int16) = with get() = 10 member x.toBytes() = - let buf = ByteBuffer.Create IMAGE_RELOCATION.Width + use buf = ByteBuffer.Create IMAGE_RELOCATION.Width buf.EmitInt32 virtualAddress buf.EmitInt32 symbolTableIndex buf.EmitUInt16 (uint16 ty) - buf.Close() + buf.GetMemory().ToArray() let bytesToIR (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_RELOCATION.Width then @@ -328,14 +328,14 @@ type IMAGE_RESOURCE_DIRECTORY(c: int32, tds: int32, mjv: int16, mnv: int16, nne: static member Width = 16 member x.toBytes () = - let buf = ByteBuffer.Create IMAGE_RESOURCE_DIRECTORY.Width + use buf = ByteBuffer.Create IMAGE_RESOURCE_DIRECTORY.Width buf.EmitInt32 characteristics buf.EmitInt32 timeDateStamp buf.EmitUInt16 (uint16 majorVersion) buf.EmitUInt16 (uint16 minorVersion) buf.EmitUInt16 (uint16 numberOfNamedEntries) buf.EmitUInt16 (uint16 numberOfIdEntries) - buf.Close() + buf.GetMemory().ToArray() let bytesToIRD (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_RESOURCE_DIRECTORY.Width then @@ -368,10 +368,10 @@ type IMAGE_RESOURCE_DIRECTORY_ENTRY(n: int32, o: int32) = static member Width = 8 member x.toBytes () = - let buf = ByteBuffer.Create IMAGE_RESOURCE_DIRECTORY_ENTRY.Width + use buf = ByteBuffer.Create IMAGE_RESOURCE_DIRECTORY_ENTRY.Width buf.EmitInt32 name buf.EmitInt32 offset - buf.Close() + buf.GetMemory().ToArray() let bytesToIRDE (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_RESOURCE_DIRECTORY_ENTRY.Width then @@ -466,7 +466,7 @@ type ResFormatHeader() = static member Width = 32 member x.toBytes() = - let buf = ByteBuffer.Create ResFormatHeader.Width + use buf = ByteBuffer.Create ResFormatHeader.Width buf.EmitInt32 dwDataSize buf.EmitInt32 dwHeaderSize buf.EmitInt32 dwTypeID @@ -476,7 +476,7 @@ type ResFormatHeader() = buf.EmitUInt16 (uint16 wLangID) buf.EmitInt32 dwVersion buf.EmitInt32 dwCharacteristics - buf.Close() + buf.GetMemory().ToArray() type ResFormatNode(tid: int32, nid: int32, lid: int32, dataOffset: int32, pbLinkedResource: byte[]) = let mutable resHdr = ResFormatHeader() diff --git a/src/fsharp/absil/ilwrite.fs b/src/fsharp/absil/ilwrite.fs index 6bb374edad4..bc4efdc1b0c 100644 --- a/src/fsharp/absil/ilwrite.fs +++ b/src/fsharp/absil/ilwrite.fs @@ -47,7 +47,7 @@ let dw0 n = byte (n &&& 0xFFL) let bitsOfSingle (x: float32) = System.BitConverter.ToInt32(System.BitConverter.GetBytes x, 0) let bitsOfDouble (x: float) = System.BitConverter.DoubleToInt64Bits x -let emitBytesViaBuffer f = let bb = ByteBuffer.Create 10 in f bb; bb.Close() +let emitBytesViaBuffer f = use bb = ByteBuffer.Create 10 in f bb; bb.GetMemory().ToArray() /// Alignment and padding let align alignment n = ((n + alignment - 1) / alignment) * alignment diff --git a/src/fsharp/service/ServiceLexing.fs b/src/fsharp/service/ServiceLexing.fs index f84b53343fc..1de7f74607a 100644 --- a/src/fsharp/service/ServiceLexing.fs +++ b/src/fsharp/service/ServiceLexing.fs @@ -706,7 +706,7 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, | LexCont.String (ifdefs, stringNest, style, kind, m) -> lexargs.ifdefStack <- ifdefs lexargs.stringNest <- stringNest - let buf = ByteBuffer.Create 100 + use buf = ByteBuffer.Create 100 let args = (buf, LexerStringFinisher.Default, m, kind, lexargs) match style with | LexerStringStyle.SingleQuote -> Lexer.singleQuoteString args skip lexbuf diff --git a/src/fsharp/utils/FileSystem.fs b/src/fsharp/utils/FileSystem.fs index 2fae0ad8766..7f3761127ef 100644 --- a/src/fsharp/utils/FileSystem.fs +++ b/src/fsharp/utils/FileSystem.fs @@ -3,7 +3,9 @@ namespace FSharp.Compiler.IO open System open System.IO open System.IO.MemoryMappedFiles +open System.Buffers open System.Reflection +open System.Threading open System.Runtime.InteropServices open FSharp.NativeInterop open Internal.Utilities.Library @@ -755,27 +757,45 @@ type internal ByteStream = type internal ByteBuffer = - { mutable bbArray: byte[] + { useArrayPool: bool + mutable isDisposed: bool + mutable bbArray: byte[] mutable bbCurrent: int } - member buf.Ensure newSize = + member inline private buf.CheckDisposed() = + if buf.isDisposed then + raise(ObjectDisposedException(nameof(ByteBuffer))) + + member private buf.Ensure newSize = let oldBufSize = buf.bbArray.Length if newSize > oldBufSize then let old = buf.bbArray - buf.bbArray <- Bytes.zeroCreate (max newSize (oldBufSize * 2)) + buf.bbArray <- + if buf.useArrayPool then + ArrayPool.Shared.Rent (max newSize (oldBufSize * 2)) + else + Bytes.sub old 0 (max newSize (oldBufSize * 2)) Bytes.blit old 0 buf.bbArray 0 buf.bbCurrent + if buf.useArrayPool then + ArrayPool.Shared.Return old - member buf.Close () = Bytes.sub buf.bbArray 0 buf.bbCurrent + member buf.GetMemory() = + buf.CheckDisposed() + ReadOnlyMemory(buf.bbArray, 0, buf.bbCurrent) member buf.EmitIntAsByte (i:int) = + buf.CheckDisposed() let newSize = buf.bbCurrent + 1 buf.Ensure newSize buf.bbArray.[buf.bbCurrent] <- byte i buf.bbCurrent <- newSize - member buf.EmitByte (b:byte) = buf.EmitIntAsByte (int b) + member buf.EmitByte (b:byte) = + buf.CheckDisposed() + buf.EmitIntAsByte (int b) member buf.EmitIntsAsBytes (arr:int[]) = + buf.CheckDisposed() let n = arr.Length let newSize = buf.bbCurrent + n buf.Ensure newSize @@ -786,52 +806,90 @@ type internal ByteBuffer = buf.bbCurrent <- newSize member bb.FixupInt32 pos value = + bb.CheckDisposed() bb.bbArray.[pos] <- (Bytes.b0 value |> byte) bb.bbArray.[pos + 1] <- (Bytes.b1 value |> byte) bb.bbArray.[pos + 2] <- (Bytes.b2 value |> byte) bb.bbArray.[pos + 3] <- (Bytes.b3 value |> byte) member buf.EmitInt32 n = + buf.CheckDisposed() let newSize = buf.bbCurrent + 4 buf.Ensure newSize buf.FixupInt32 buf.bbCurrent n buf.bbCurrent <- newSize member buf.EmitBytes (i:byte[]) = + buf.CheckDisposed() let n = i.Length let newSize = buf.bbCurrent + n buf.Ensure newSize Bytes.blit i 0 buf.bbArray buf.bbCurrent n buf.bbCurrent <- newSize + member buf.EmitMemory (i:ReadOnlyMemory) = + buf.CheckDisposed() + let n = i.Length + let newSize = buf.bbCurrent + n + buf.Ensure newSize + i.CopyTo(Memory(buf.bbArray, buf.bbCurrent, n)) + buf.bbCurrent <- newSize + member buf.EmitByteMemory (i:ReadOnlyByteMemory) = + buf.CheckDisposed() let n = i.Length let newSize = buf.bbCurrent + n buf.Ensure newSize i.Copy(0, buf.bbArray, buf.bbCurrent, n) buf.bbCurrent <- newSize + member buf.EmitByteBuffer (i:ByteBuffer) = + buf.CheckDisposed() + let n = i.Position + let newSize = buf.bbCurrent + n + buf.Ensure newSize + Bytes.blit i.bbArray 0 buf.bbArray buf.bbCurrent n + buf.bbCurrent <- newSize + member buf.EmitInt32AsUInt16 n = + buf.CheckDisposed() let newSize = buf.bbCurrent + 2 buf.Ensure newSize buf.bbArray.[buf.bbCurrent] <- (Bytes.b0 n |> byte) buf.bbArray.[buf.bbCurrent + 1] <- (Bytes.b1 n |> byte) buf.bbCurrent <- newSize - member buf.EmitBoolAsByte (b:bool) = buf.EmitIntAsByte (if b then 1 else 0) + member buf.EmitBoolAsByte (b:bool) = + buf.CheckDisposed() + buf.EmitIntAsByte (if b then 1 else 0) - member buf.EmitUInt16 (x:uint16) = buf.EmitInt32AsUInt16 (int32 x) + member buf.EmitUInt16 (x:uint16) = + buf.CheckDisposed() + buf.EmitInt32AsUInt16 (int32 x) member buf.EmitInt64 x = + buf.CheckDisposed() buf.EmitInt32 (Bytes.dWw0 x) buf.EmitInt32 (Bytes.dWw1 x) - member buf.Position = buf.bbCurrent + member buf.Position = + buf.CheckDisposed() + buf.bbCurrent - static member Create sz = - { bbArray = Bytes.zeroCreate sz + static member Create(sz, useArrayPool) = + { useArrayPool = defaultArg useArrayPool false + isDisposed = false + bbArray = ArrayPool.Shared.Rent sz bbCurrent = 0 } + interface IDisposable with + + member this.Dispose() = + if not this.isDisposed then + this.isDisposed <- true + if this.useArrayPool then + ArrayPool.Shared.Return this.bbArray + [] type ByteStorage(getByteMemory: unit -> ReadOnlyByteMemory) = diff --git a/src/fsharp/utils/FileSystem.fsi b/src/fsharp/utils/FileSystem.fsi index 37c2b1a66b1..a1775b2906f 100644 --- a/src/fsharp/utils/FileSystem.fsi +++ b/src/fsharp/utils/FileSystem.fsi @@ -335,14 +335,19 @@ type internal ByteStream = #endif /// Imperative buffers and streams of byte[] +/// Not thread safe. [] type internal ByteBuffer = - member Close : unit -> byte[] + interface IDisposable + + member GetMemory : unit -> ReadOnlyMemory member EmitIntAsByte : int -> unit member EmitIntsAsBytes : int[] -> unit member EmitByte : byte -> unit member EmitBytes : byte[] -> unit + member EmitMemory : ReadOnlyMemory -> unit member EmitByteMemory : ReadOnlyByteMemory -> unit + member EmitByteBuffer : ByteBuffer -> unit member EmitInt32 : int32 -> unit member EmitInt64 : int64 -> unit member FixupInt32 : pos: int -> value: int32 -> unit @@ -350,7 +355,7 @@ type internal ByteBuffer = member EmitBoolAsByte : bool -> unit member EmitUInt16 : uint16 -> unit member Position : int - static member Create : int -> ByteBuffer + static member Create : int * ?useArrayPool: bool -> ByteBuffer [] type internal ByteStorage = From 601f236888b25ba55b14fd36b121266a473b74d9 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Wed, 9 Jun 2021 13:27:39 -0700 Subject: [PATCH 02/16] Using ArrayPool --- src/fsharp/QuotationPickler.fs | 4 +-- src/fsharp/TypedTreePickle.fs | 57 +++++++++++++++++++-------------- src/fsharp/absil/ilsupp.fs | 2 +- src/fsharp/absil/ilwrite.fs | 43 ++++++++++++++++--------- src/fsharp/lexhelp.fs | 14 ++++---- src/fsharp/utils/FileSystem.fsi | 29 +++++++++++++++++ 6 files changed, 99 insertions(+), 50 deletions(-) diff --git a/src/fsharp/QuotationPickler.fs b/src/fsharp/QuotationPickler.fs index b69a16cda5a..2b37797b55f 100644 --- a/src/fsharp/QuotationPickler.fs +++ b/src/fsharp/QuotationPickler.fs @@ -383,9 +383,9 @@ module SimplePickle = ostrings=Table<_>.Create() } let phase2bytes = p_tup2 (p_list prim_pstring) p_memory phase2data st2 - st2.os + st2.os.GetMemory() - let finalBytes = phase2bytes.GetMemory().ToArray() + let finalBytes = phase2bytes.ToArray() (st1.os :> IDisposable).Dispose() (st2.os :> IDisposable).Dispose() finalBytes diff --git a/src/fsharp/TypedTreePickle.fs b/src/fsharp/TypedTreePickle.fs index 9e48b1c23fb..63df18a9309 100644 --- a/src/fsharp/TypedTreePickle.fs +++ b/src/fsharp/TypedTreePickle.fs @@ -213,6 +213,11 @@ let p_bytes (s: byte[]) st = p_int32 len st st.os.EmitBytes s +let p_memory (s: System.ReadOnlyMemory) st = + let len = s.Length + p_int32 len st + st.os.EmitMemory s + let p_prim_string (s: string) st = let bytes = Encoding.UTF8.GetBytes s let len = bytes.Length @@ -776,9 +781,8 @@ let p_encoded_anoninfo x st = p_int x st let p_simpletyp x st = p_int (encode_simpletyp st.occus st.ostrings st.onlerefs st.osimpletys st.oscope x) st let pickleObjWithDanglingCcus inMem file g scope p x = - let ccuNameTab, (ntycons, ntypars, nvals, nanoninfos), stringTab, pubpathTab, nlerefTab, simpleTyTab, phase1bytes = - let st1 = - { os = ByteBuffer.Create 100000 + let st1 = + { os = ByteBuffer.Create(100000, useArrayPool = true) oscope=scope occus= Table<_>.Create "occus" oentities=NodeOutTable<_, _>.Create((fun (tc: Tycon) -> tc.Stamp), (fun tc -> tc.LogicalName), (fun tc -> tc.Range), (fun osgn -> osgn), "otycons") @@ -793,31 +797,32 @@ let pickleObjWithDanglingCcus inMem file g scope p x = ofile=file oInMem=inMem isStructThisArgPos = false} + let ccuNameTab, (ntycons, ntypars, nvals, nanoninfos), stringTab, pubpathTab, nlerefTab, simpleTyTab, phase1bytes = p x st1 let sizes = st1.oentities.Size, st1.otypars.Size, st1.ovals.Size, st1.oanoninfos.Size - st1.occus, sizes, st1.ostrings, st1.opubpaths, st1.onlerefs, st1.osimpletys, st1.os.Close() - + st1.occus, sizes, st1.ostrings, st1.opubpaths, st1.onlerefs, st1.osimpletys, st1.os.GetMemory() + + let st2 = + { os = ByteBuffer.Create(100000, useArrayPool = true) + oscope=scope + occus= Table<_>.Create "occus (fake)" + oentities=NodeOutTable<_, _>.Create((fun (tc: Tycon) -> tc.Stamp), (fun tc -> tc.LogicalName), (fun tc -> tc.Range), (fun osgn -> osgn), "otycons") + otypars=NodeOutTable<_, _>.Create((fun (tp: Typar) -> tp.Stamp), (fun tp -> tp.DisplayName), (fun tp -> tp.Range), (fun osgn -> osgn), "otypars") + ovals=NodeOutTable<_, _>.Create((fun (v: Val) -> v.Stamp), (fun v -> v.LogicalName), (fun v -> v.Range), (fun osgn -> osgn), "ovals") + oanoninfos=NodeOutTable<_, _>.Create((fun (v: AnonRecdTypeInfo) -> v.Stamp), (fun v -> string v.Stamp), (fun _ -> range0), id, "oanoninfos") + ostrings=Table<_>.Create "ostrings (fake)" + opubpaths=Table<_>.Create "opubpaths (fake)" + onlerefs=Table<_>.Create "onlerefs (fake)" + osimpletys=Table<_>.Create "osimpletys (fake)" + oglobals=g + ofile=file + oInMem=inMem + isStructThisArgPos = false } let phase2bytes = - let st2 = - { os = ByteBuffer.Create 100000 - oscope=scope - occus= Table<_>.Create "occus (fake)" - oentities=NodeOutTable<_, _>.Create((fun (tc: Tycon) -> tc.Stamp), (fun tc -> tc.LogicalName), (fun tc -> tc.Range), (fun osgn -> osgn), "otycons") - otypars=NodeOutTable<_, _>.Create((fun (tp: Typar) -> tp.Stamp), (fun tp -> tp.DisplayName), (fun tp -> tp.Range), (fun osgn -> osgn), "otypars") - ovals=NodeOutTable<_, _>.Create((fun (v: Val) -> v.Stamp), (fun v -> v.LogicalName), (fun v -> v.Range), (fun osgn -> osgn), "ovals") - oanoninfos=NodeOutTable<_, _>.Create((fun (v: AnonRecdTypeInfo) -> v.Stamp), (fun v -> string v.Stamp), (fun _ -> range0), id, "oanoninfos") - ostrings=Table<_>.Create "ostrings (fake)" - opubpaths=Table<_>.Create "opubpaths (fake)" - onlerefs=Table<_>.Create "onlerefs (fake)" - osimpletys=Table<_>.Create "osimpletys (fake)" - oglobals=g - ofile=file - oInMem=inMem - isStructThisArgPos = false } p_array p_encoded_ccuref ccuNameTab.AsArray st2 // Add a 4th integer indicated by a negative 1st integer let z1 = if nanoninfos > 0 then -ntycons-1 else ntycons @@ -830,11 +835,15 @@ let pickleObjWithDanglingCcus inMem file g scope p x = (p_array p_encoded_pubpath) (p_array p_encoded_nleref) (p_array p_encoded_simpletyp) - p_bytes + p_memory (stringTab.AsArray, pubpathTab.AsArray, nlerefTab.AsArray, simpleTyTab.AsArray, phase1bytes) st2 - st2.os.Close() - phase2bytes + st2.os.GetMemory() + + let finalBytes = phase2bytes.ToArray() + (st1.os :> System.IDisposable).Dispose() + (st2.os :> System.IDisposable).Dispose() + finalBytes let check (ilscope: ILScopeRef) (inMap : NodeInTable<_, _>) = for i = 0 to inMap.Count - 1 do diff --git a/src/fsharp/absil/ilsupp.fs b/src/fsharp/absil/ilsupp.fs index 9ad0598f5c5..43a975b17ac 100644 --- a/src/fsharp/absil/ilsupp.fs +++ b/src/fsharp/absil/ilsupp.fs @@ -401,7 +401,7 @@ type IMAGE_RESOURCE_DATA_ENTRY(o: int32, s: int32, c: int32, r: int32) = static member Width = 16 member x.toBytes() = - let buf = ByteBuffer.Create IMAGE_RESOURCE_DATA_ENTRY.Width + use buf = ByteBuffer.Create IMAGE_RESOURCE_DATA_ENTRY.Width buf.EmitInt32 offsetToData buf.EmitInt32 size buf.EmitInt32 codePage diff --git a/src/fsharp/absil/ilwrite.fs b/src/fsharp/absil/ilwrite.fs index bc4efdc1b0c..8bc365000d9 100644 --- a/src/fsharp/absil/ilwrite.fs +++ b/src/fsharp/absil/ilwrite.fs @@ -2,6 +2,7 @@ module internal FSharp.Compiler.AbstractIL.ILBinaryWriter +open System open System.Collections.Generic open System.IO @@ -543,10 +544,16 @@ type cenv = cenv.codeChunks.EmitBytes code cenv.nextCodeAddr <- cenv.nextCodeAddr + code.Length - member cenv.GetCode() = cenv.codeChunks.Close() + member cenv.GetCode() = cenv.codeChunks.GetMemory().ToArray() override x.ToString() = "" + interface IDisposable with + member this.Dispose() = + (this.codeChunks :> IDisposable).Dispose() + (this.data :> IDisposable).Dispose() + (this.resources :> IDisposable).Dispose() + let FindOrAddSharedRow (cenv: cenv) tbl x = cenv.GetTable(tbl).FindOrAddSharedEntry x // Shared rows must be hash-cons'd to be made unique (no duplicates according to contents) @@ -1465,6 +1472,10 @@ type CodeBuffer = mutable seh: ExceptionClauseSpec list seqpoints: ResizeArray } + interface IDisposable with + member this.Dispose() = + (this.code :> IDisposable).Dispose() + static member Create _nm = { seh = [] code= ByteBuffer.Create 200 @@ -1534,7 +1545,7 @@ module Codebuf = let applyBrFixups (origCode : byte[]) origExnClauses origReqdStringFixups (origAvailBrFixups: Dictionary) origReqdBrFixups origSeqPoints origScopes = let orderedOrigReqdBrFixups = origReqdBrFixups |> List.sortBy (fun (_, fixupLoc, _) -> fixupLoc) - let newCode = ByteBuffer.Create origCode.Length + use newCode = ByteBuffer.Create origCode.Length // Copy over all the code, working out whether the branches will be short // or long and adjusting the branch destinations. Record an adjust function to adjust all the other @@ -1639,7 +1650,7 @@ module Codebuf = let (origStartOfNoBranchBlock, _, newStartOfNoBranchBlock) = arr.[i] addr - (origStartOfNoBranchBlock - newStartOfNoBranchBlock) - newCode.Close(), + newCode.GetMemory().ToArray(), !newReqdBrFixups, adjuster @@ -2138,9 +2149,9 @@ module Codebuf = localsTree let EmitTopCode cenv localSigs env nm code = - let codebuf = CodeBuffer.Create nm + use codebuf = CodeBuffer.Create nm let origScopes = emitCode cenv localSigs codebuf env code - let origCode = codebuf.code.Close() + let origCode = codebuf.code.GetMemory().ToArray() let origExnClauses = List.rev codebuf.seh let origReqdStringFixups = codebuf.reqdStringFixupsInMethod let origAvailBrFixups = codebuf.availBrFixups @@ -2179,7 +2190,7 @@ let GenILMethodBody mname cenv env (il: ILMethodBody) = let requiredStringFixups, seh, code, seqpoints, scopes = Codebuf.EmitTopCode cenv localSigs env mname il.Code let codeSize = code.Length - let methbuf = ByteBuffer.Create (codeSize * 3) + use methbuf = ByteBuffer.Create (codeSize * 3) // Do we use the tiny format? if isNil il.Locals && il.MaxStack <= 8 && isNil seh && codeSize < 64 then // Use Tiny format @@ -2189,7 +2200,7 @@ let GenILMethodBody mname cenv env (il: ILMethodBody) = methbuf.EmitByte (byte codeSize <<< 2 ||| e_CorILMethod_TinyFormat) methbuf.EmitBytes code methbuf.EmitPadding codePadding - 0x0, (requiredStringFixups', methbuf.Close()), seqpoints, scopes + 0x0, (requiredStringFixups', methbuf.GetMemory().ToArray()), seqpoints, scopes else // Use Fat format let flags = @@ -2263,7 +2274,7 @@ let GenILMethodBody mname cenv env (il: ILMethodBody) = let requiredStringFixups' = (12, requiredStringFixups) - localToken, (requiredStringFixups', methbuf.Close()), seqpoints, scopes + localToken, (requiredStringFixups', methbuf.GetMemory().ToArray()), seqpoints, scopes // -------------------------------------------------------------------- // ILFieldDef --> FieldDef Row @@ -2862,7 +2873,7 @@ let GenModule (cenv : cenv) (modul: ILModuleDef) = let generateIL requiredDataFixups (desiredMetadataVersion, generatePdb, ilg : ILGlobals, emitTailcalls, deterministic, showTimes) (m : ILModuleDef) cilStartAddress normalizeAssemblyRefs = let isDll = m.IsDLL - let cenv = + use cenv = { emitTailcalls=emitTailcalls deterministic = deterministic showTimes=showTimes @@ -2870,7 +2881,7 @@ let generateIL requiredDataFixups (desiredMetadataVersion, generatePdb, ilg : IL desiredMetadataVersion=desiredMetadataVersion requiredDataFixups= requiredDataFixups requiredStringFixups = [] - codeChunks=ByteBuffer.Create 40000 + codeChunks=ByteBuffer.Create(40000, useArrayPool = true) nextCodeAddr = cilStartAddress data = ByteBuffer.Create 200 resources = ByteBuffer.Create 200 @@ -2959,8 +2970,8 @@ let generateIL requiredDataFixups (desiredMetadataVersion, generatePdb, ilg : IL getUncodedToken TableNames.Event (cenv.eventDefs.GetTableEntry (EventKey (tidx, ed.Name)))) } reportTime cenv.showTimes "Finalize Module Generation Results" // New return the results - let data = cenv.data.Close() - let resources = cenv.resources.Close() + let data = cenv.data.GetMemory().ToArray() + let resources = cenv.resources.GetMemory().ToArray() (strings, userStrings, blobs, guids, tables, entryPointToken, code, cenv.requiredStringFixups, data, resources, pdbData, mappings) @@ -3230,7 +3241,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca codedBigness 2 TableNames.AssemblyRef || codedBigness 2 TableNames.TypeRef - let tablesBuf = ByteBuffer.Create 20000 + use tablesBuf = ByteBuffer.Create 20000 // Now the coded tables themselves - first the schemata header tablesBuf.EmitIntsAsBytes @@ -3287,7 +3298,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca | _ when t <= RowElementTags.ResolutionScopeMax -> tablesBuf.EmitZTaggedIndex (t - RowElementTags.ResolutionScopeMin) 2 rsBigness n | _ -> failwith "invalid tag in row element" - tablesBuf.Close() + tablesBuf.GetMemory().ToArray() reportTime showTimes "Write Tables to tablebuf" @@ -3309,7 +3320,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca reportTime showTimes "Layout Metadata" let metadata, guidStart = - let mdbuf = ByteBuffer.Create 500000 + use mdbuf = ByteBuffer.Create 500000 mdbuf.EmitIntsAsBytes [| 0x42; 0x53; 0x4a; 0x42 // Magic signature 0x01; 0x00 // Major version @@ -3378,7 +3389,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca mdbuf.EmitIntAsByte 0x00 reportTime showTimes "Write Blob Stream" // Done - close the buffer and return the result. - mdbuf.Close(), guidStart + mdbuf.GetMemory().ToArray(), guidStart // Now we know the user string tables etc. we can fixup the diff --git a/src/fsharp/lexhelp.fs b/src/fsharp/lexhelp.fs index 8fa659de6ee..3deb962e9c9 100644 --- a/src/fsharp/lexhelp.fs +++ b/src/fsharp/lexhelp.fs @@ -101,12 +101,12 @@ let usingLexbufForParsing (lexbuf:Lexbuf, filename) f = //----------------------------------------------------------------------- let stringBufferAsString (buf: ByteBuffer) = - let buf = buf.Close() + let buf = buf.GetMemory() if buf.Length % 2 <> 0 then failwith "Expected even number of bytes" let chars : char[] = Array.zeroCreate (buf.Length/2) for i = 0 to (buf.Length/2) - 1 do - let hi = buf.[i*2+1] - let lo = buf.[i*2] + let hi = buf.Span.[i*2+1] + let lo = buf.Span.[i*2] let c = char (((int hi) * 256) + (int lo)) chars.[i] <- c System.String(chars) @@ -117,8 +117,8 @@ let stringBufferAsString (buf: ByteBuffer) = /// we just take every second byte we stored. Note all bytes > 127 should have been /// stored using addIntChar let stringBufferAsBytes (buf: ByteBuffer) = - let bytes = buf.Close() - Array.init (bytes.Length / 2) (fun i -> bytes.[i*2]) + let bytes = buf.GetMemory() + Array.init (bytes.Length / 2) (fun i -> bytes.Span.[i*2]) [] type LexerStringFinisherContext = @@ -185,10 +185,10 @@ let addByteChar buf (c:char) = addIntChar buf (int32 c % 256) /// Sanity check that high bytes are zeros. Further check each low byte <= 127 let stringBufferIsBytes (buf: ByteBuffer) = - let bytes = buf.Close() + let bytes = buf.GetMemory() let mutable ok = true for i = 0 to bytes.Length / 2-1 do - if bytes.[i*2+1] <> 0uy then ok <- false + if bytes.Span.[i*2+1] <> 0uy then ok <- false ok let newline (lexbuf:LexBuffer<_>) = diff --git a/src/fsharp/utils/FileSystem.fsi b/src/fsharp/utils/FileSystem.fsi index a1775b2906f..87a5a853d2b 100644 --- a/src/fsharp/utils/FileSystem.fsi +++ b/src/fsharp/utils/FileSystem.fsi @@ -8,6 +8,7 @@ open System.IO.MemoryMappedFiles open System.Reflection open System.Runtime.InteropServices open System.Text +open System.Runtime.CompilerServices open FSharp.NativeInterop @@ -340,20 +341,48 @@ type internal ByteStream = type internal ByteBuffer = interface IDisposable + [] member GetMemory : unit -> ReadOnlyMemory + + [] member EmitIntAsByte : int -> unit + + [] member EmitIntsAsBytes : int[] -> unit + + [] member EmitByte : byte -> unit + + [] member EmitBytes : byte[] -> unit + + [] member EmitMemory : ReadOnlyMemory -> unit + + [] member EmitByteMemory : ReadOnlyByteMemory -> unit + + [] member EmitByteBuffer : ByteBuffer -> unit + + [] member EmitInt32 : int32 -> unit + + [] member EmitInt64 : int64 -> unit + + [] member FixupInt32 : pos: int -> value: int32 -> unit + + [] member EmitInt32AsUInt16 : int32 -> unit + + [] member EmitBoolAsByte : bool -> unit + + [] member EmitUInt16 : uint16 -> unit + member Position : int static member Create : int * ?useArrayPool: bool -> ByteBuffer From fcc742b8246cd3abd0783d0d6c60e55e23ee70e6 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Wed, 9 Jun 2021 13:33:31 -0700 Subject: [PATCH 03/16] Renaming GetMemory to AsMemory --- src/fsharp/absil/ilsupp.fs | 14 +++++++------- src/fsharp/absil/ilwrite.fs | 20 ++++++++++---------- src/fsharp/lexhelp.fs | 6 +++--- src/fsharp/utils/FileSystem.fs | 2 +- src/fsharp/utils/FileSystem.fsi | 2 +- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/fsharp/absil/ilsupp.fs b/src/fsharp/absil/ilsupp.fs index 43a975b17ac..cf2658bd31a 100644 --- a/src/fsharp/absil/ilsupp.fs +++ b/src/fsharp/absil/ilsupp.fs @@ -99,7 +99,7 @@ type IMAGE_FILE_HEADER (m: int16, secs: int16, tds: int32, ptst: int32, nos: int buf.EmitInt32 numberOfSymbols buf.EmitUInt16 (uint16 sizeOfOptionalHeader) buf.EmitUInt16 (uint16 characteristics) - buf.GetMemory().ToArray() + buf.AsMemory().ToArray() let bytesToIFH (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_FILE_HEADER.Width then @@ -183,7 +183,7 @@ type IMAGE_SECTION_HEADER(n: int64, ai: int32, va: int32, srd: int32, prd: int32 buf.EmitUInt16 (uint16 numberOfRelocations) buf.EmitUInt16 (uint16 numberOfLineNumbers) buf.EmitInt32 characteristics - buf.GetMemory().ToArray() + buf.AsMemory().ToArray() let bytesToISH (buffer: byte[]) (offset: int) = @@ -243,7 +243,7 @@ type IMAGE_SYMBOL(n: int64, v: int32, sn: int16, t: int16, sc: byte, nas: byte) buf.EmitUInt16 (uint16 stype) buf.EmitByte storageClass buf.EmitByte numberOfAuxSymbols - buf.GetMemory().ToArray() + buf.AsMemory().ToArray() let bytesToIS (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_SYMBOL.Width then @@ -284,7 +284,7 @@ type IMAGE_RELOCATION(va: int32, sti: int32, t: int16) = buf.EmitInt32 virtualAddress buf.EmitInt32 symbolTableIndex buf.EmitUInt16 (uint16 ty) - buf.GetMemory().ToArray() + buf.AsMemory().ToArray() let bytesToIR (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_RELOCATION.Width then @@ -335,7 +335,7 @@ type IMAGE_RESOURCE_DIRECTORY(c: int32, tds: int32, mjv: int16, mnv: int16, nne: buf.EmitUInt16 (uint16 minorVersion) buf.EmitUInt16 (uint16 numberOfNamedEntries) buf.EmitUInt16 (uint16 numberOfIdEntries) - buf.GetMemory().ToArray() + buf.AsMemory().ToArray() let bytesToIRD (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_RESOURCE_DIRECTORY.Width then @@ -371,7 +371,7 @@ type IMAGE_RESOURCE_DIRECTORY_ENTRY(n: int32, o: int32) = use buf = ByteBuffer.Create IMAGE_RESOURCE_DIRECTORY_ENTRY.Width buf.EmitInt32 name buf.EmitInt32 offset - buf.GetMemory().ToArray() + buf.AsMemory().ToArray() let bytesToIRDE (buffer: byte[]) (offset: int) = if (buffer.Length - offset) < IMAGE_RESOURCE_DIRECTORY_ENTRY.Width then @@ -476,7 +476,7 @@ type ResFormatHeader() = buf.EmitUInt16 (uint16 wLangID) buf.EmitInt32 dwVersion buf.EmitInt32 dwCharacteristics - buf.GetMemory().ToArray() + buf.AsMemory().ToArray() type ResFormatNode(tid: int32, nid: int32, lid: int32, dataOffset: int32, pbLinkedResource: byte[]) = let mutable resHdr = ResFormatHeader() diff --git a/src/fsharp/absil/ilwrite.fs b/src/fsharp/absil/ilwrite.fs index 8bc365000d9..e6ccc4b6f9f 100644 --- a/src/fsharp/absil/ilwrite.fs +++ b/src/fsharp/absil/ilwrite.fs @@ -48,7 +48,7 @@ let dw0 n = byte (n &&& 0xFFL) let bitsOfSingle (x: float32) = System.BitConverter.ToInt32(System.BitConverter.GetBytes x, 0) let bitsOfDouble (x: float) = System.BitConverter.DoubleToInt64Bits x -let emitBytesViaBuffer f = use bb = ByteBuffer.Create 10 in f bb; bb.GetMemory().ToArray() +let emitBytesViaBuffer f = use bb = ByteBuffer.Create 10 in f bb; bb.AsMemory().ToArray() /// Alignment and padding let align alignment n = ((n + alignment - 1) / alignment) * alignment @@ -544,7 +544,7 @@ type cenv = cenv.codeChunks.EmitBytes code cenv.nextCodeAddr <- cenv.nextCodeAddr + code.Length - member cenv.GetCode() = cenv.codeChunks.GetMemory().ToArray() + member cenv.GetCode() = cenv.codeChunks.AsMemory().ToArray() override x.ToString() = "" @@ -1650,7 +1650,7 @@ module Codebuf = let (origStartOfNoBranchBlock, _, newStartOfNoBranchBlock) = arr.[i] addr - (origStartOfNoBranchBlock - newStartOfNoBranchBlock) - newCode.GetMemory().ToArray(), + newCode.AsMemory().ToArray(), !newReqdBrFixups, adjuster @@ -2151,7 +2151,7 @@ module Codebuf = let EmitTopCode cenv localSigs env nm code = use codebuf = CodeBuffer.Create nm let origScopes = emitCode cenv localSigs codebuf env code - let origCode = codebuf.code.GetMemory().ToArray() + let origCode = codebuf.code.AsMemory().ToArray() let origExnClauses = List.rev codebuf.seh let origReqdStringFixups = codebuf.reqdStringFixupsInMethod let origAvailBrFixups = codebuf.availBrFixups @@ -2200,7 +2200,7 @@ let GenILMethodBody mname cenv env (il: ILMethodBody) = methbuf.EmitByte (byte codeSize <<< 2 ||| e_CorILMethod_TinyFormat) methbuf.EmitBytes code methbuf.EmitPadding codePadding - 0x0, (requiredStringFixups', methbuf.GetMemory().ToArray()), seqpoints, scopes + 0x0, (requiredStringFixups', methbuf.AsMemory().ToArray()), seqpoints, scopes else // Use Fat format let flags = @@ -2274,7 +2274,7 @@ let GenILMethodBody mname cenv env (il: ILMethodBody) = let requiredStringFixups' = (12, requiredStringFixups) - localToken, (requiredStringFixups', methbuf.GetMemory().ToArray()), seqpoints, scopes + localToken, (requiredStringFixups', methbuf.AsMemory().ToArray()), seqpoints, scopes // -------------------------------------------------------------------- // ILFieldDef --> FieldDef Row @@ -2970,8 +2970,8 @@ let generateIL requiredDataFixups (desiredMetadataVersion, generatePdb, ilg : IL getUncodedToken TableNames.Event (cenv.eventDefs.GetTableEntry (EventKey (tidx, ed.Name)))) } reportTime cenv.showTimes "Finalize Module Generation Results" // New return the results - let data = cenv.data.GetMemory().ToArray() - let resources = cenv.resources.GetMemory().ToArray() + let data = cenv.data.AsMemory().ToArray() + let resources = cenv.resources.AsMemory().ToArray() (strings, userStrings, blobs, guids, tables, entryPointToken, code, cenv.requiredStringFixups, data, resources, pdbData, mappings) @@ -3298,7 +3298,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca | _ when t <= RowElementTags.ResolutionScopeMax -> tablesBuf.EmitZTaggedIndex (t - RowElementTags.ResolutionScopeMin) 2 rsBigness n | _ -> failwith "invalid tag in row element" - tablesBuf.GetMemory().ToArray() + tablesBuf.AsMemory().ToArray() reportTime showTimes "Write Tables to tablebuf" @@ -3389,7 +3389,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca mdbuf.EmitIntAsByte 0x00 reportTime showTimes "Write Blob Stream" // Done - close the buffer and return the result. - mdbuf.GetMemory().ToArray(), guidStart + mdbuf.AsMemory().ToArray(), guidStart // Now we know the user string tables etc. we can fixup the diff --git a/src/fsharp/lexhelp.fs b/src/fsharp/lexhelp.fs index 3deb962e9c9..7de63e244a1 100644 --- a/src/fsharp/lexhelp.fs +++ b/src/fsharp/lexhelp.fs @@ -101,7 +101,7 @@ let usingLexbufForParsing (lexbuf:Lexbuf, filename) f = //----------------------------------------------------------------------- let stringBufferAsString (buf: ByteBuffer) = - let buf = buf.GetMemory() + let buf = buf.AsMemory() if buf.Length % 2 <> 0 then failwith "Expected even number of bytes" let chars : char[] = Array.zeroCreate (buf.Length/2) for i = 0 to (buf.Length/2) - 1 do @@ -117,7 +117,7 @@ let stringBufferAsString (buf: ByteBuffer) = /// we just take every second byte we stored. Note all bytes > 127 should have been /// stored using addIntChar let stringBufferAsBytes (buf: ByteBuffer) = - let bytes = buf.GetMemory() + let bytes = buf.AsMemory() Array.init (bytes.Length / 2) (fun i -> bytes.Span.[i*2]) [] @@ -185,7 +185,7 @@ let addByteChar buf (c:char) = addIntChar buf (int32 c % 256) /// Sanity check that high bytes are zeros. Further check each low byte <= 127 let stringBufferIsBytes (buf: ByteBuffer) = - let bytes = buf.GetMemory() + let bytes = buf.AsMemory() let mutable ok = true for i = 0 to bytes.Length / 2-1 do if bytes.Span.[i*2+1] <> 0uy then ok <- false diff --git a/src/fsharp/utils/FileSystem.fs b/src/fsharp/utils/FileSystem.fs index 7f3761127ef..3f7cb9520c0 100644 --- a/src/fsharp/utils/FileSystem.fs +++ b/src/fsharp/utils/FileSystem.fs @@ -779,7 +779,7 @@ type internal ByteBuffer = if buf.useArrayPool then ArrayPool.Shared.Return old - member buf.GetMemory() = + member buf.AsMemory() = buf.CheckDisposed() ReadOnlyMemory(buf.bbArray, 0, buf.bbCurrent) diff --git a/src/fsharp/utils/FileSystem.fsi b/src/fsharp/utils/FileSystem.fsi index 87a5a853d2b..616dcee939b 100644 --- a/src/fsharp/utils/FileSystem.fsi +++ b/src/fsharp/utils/FileSystem.fsi @@ -342,7 +342,7 @@ type internal ByteBuffer = interface IDisposable [] - member GetMemory : unit -> ReadOnlyMemory + member AsMemory : unit -> ReadOnlyMemory [] member EmitIntAsByte : int -> unit From c407feb073897cd9d68dacdb82ad3456c54e147c Mon Sep 17 00:00:00 2001 From: Will Smith Date: Wed, 9 Jun 2021 13:34:47 -0700 Subject: [PATCH 04/16] Fixing build --- src/fsharp/IlxGen.fs | 2 +- src/fsharp/QuotationPickler.fs | 4 ++-- src/fsharp/TypedTreePickle.fs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/fsharp/IlxGen.fs b/src/fsharp/IlxGen.fs index 4e005dc7f29..6cefacd50c3 100644 --- a/src/fsharp/IlxGen.fs +++ b/src/fsharp/IlxGen.fs @@ -2029,7 +2029,7 @@ let GenConstArray cenv (cgbuf: CodeGenBuffer) eenv ilElementType (data:'a[]) (wr let g = cenv.g use buf = ByteBuffer.Create data.Length data |> Array.iter (write buf) - let bytes = buf.GetMemory().ToArray() + let bytes = buf.AsMemory().ToArray() let ilArrayType = mkILArr1DTy ilElementType if data.Length = 0 then CG.EmitInstrs cgbuf (pop 0) (Push [ilArrayType]) [ mkLdcInt32 0; I_newarr (ILArrayShape.SingleDimensional, ilElementType); ] diff --git a/src/fsharp/QuotationPickler.fs b/src/fsharp/QuotationPickler.fs index 2b37797b55f..5e0abd1052a 100644 --- a/src/fsharp/QuotationPickler.fs +++ b/src/fsharp/QuotationPickler.fs @@ -374,7 +374,7 @@ module SimplePickle = ostrings=Table<_>.Create() } let stringTab, phase1bytes = p x st1 - st1.ostrings.AsList, st1.os.GetMemory() + st1.ostrings.AsList, st1.os.AsMemory() let phase2data = (stringTab, phase1bytes) @@ -383,7 +383,7 @@ module SimplePickle = ostrings=Table<_>.Create() } let phase2bytes = p_tup2 (p_list prim_pstring) p_memory phase2data st2 - st2.os.GetMemory() + st2.os.AsMemory() let finalBytes = phase2bytes.ToArray() (st1.os :> IDisposable).Dispose() diff --git a/src/fsharp/TypedTreePickle.fs b/src/fsharp/TypedTreePickle.fs index 63df18a9309..d28f0842f35 100644 --- a/src/fsharp/TypedTreePickle.fs +++ b/src/fsharp/TypedTreePickle.fs @@ -804,7 +804,7 @@ let pickleObjWithDanglingCcus inMem file g scope p x = st1.otypars.Size, st1.ovals.Size, st1.oanoninfos.Size - st1.occus, sizes, st1.ostrings, st1.opubpaths, st1.onlerefs, st1.osimpletys, st1.os.GetMemory() + st1.occus, sizes, st1.ostrings, st1.opubpaths, st1.onlerefs, st1.osimpletys, st1.os.AsMemory() let st2 = { os = ByteBuffer.Create(100000, useArrayPool = true) @@ -838,7 +838,7 @@ let pickleObjWithDanglingCcus inMem file g scope p x = p_memory (stringTab.AsArray, pubpathTab.AsArray, nlerefTab.AsArray, simpleTyTab.AsArray, phase1bytes) st2 - st2.os.GetMemory() + st2.os.AsMemory() let finalBytes = phase2bytes.ToArray() (st1.os :> System.IDisposable).Dispose() From 680df7b11ad5a398eee7871b7cfc907c7f4e6454 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Wed, 9 Jun 2021 14:07:58 -0700 Subject: [PATCH 05/16] Returning ByteBuffer from TypedTreePickle --- src/fsharp/CompilerImports.fs | 6 ++-- src/fsharp/TypedTreePickle.fs | 5 ++- src/fsharp/TypedTreePickle.fsi | 2 +- src/fsharp/utils/FileSystem.fs | 64 +++++++++++++++++++++++---------- src/fsharp/utils/FileSystem.fsi | 4 +++ 5 files changed, 56 insertions(+), 25 deletions(-) diff --git a/src/fsharp/CompilerImports.fs b/src/fsharp/CompilerImports.fs index c730190ad0f..82b55aaff7e 100644 --- a/src/fsharp/CompilerImports.fs +++ b/src/fsharp/CompilerImports.fs @@ -90,9 +90,11 @@ let PickleToResource inMem file (g: TcGlobals) scope rName p x = let bytes = pickleObjWithDanglingCcus inMem file g scope p x let byteStorage = if inMem then - ByteStorage.FromByteArrayAndCopy(bytes, useBackingMemoryMappedFile = true) + ByteStorage.FromMemoryAndCopy(bytes.AsMemory(), useBackingMemoryMappedFile = true) else - ByteStorage.FromByteArray(bytes) + ByteStorage.FromByteArray(bytes.AsMemory().ToArray()) + + (bytes :> IDisposable).Dispose() { Name = rName Location = ILResourceLocation.Local(byteStorage) diff --git a/src/fsharp/TypedTreePickle.fs b/src/fsharp/TypedTreePickle.fs index d28f0842f35..21aefc62497 100644 --- a/src/fsharp/TypedTreePickle.fs +++ b/src/fsharp/TypedTreePickle.fs @@ -838,11 +838,10 @@ let pickleObjWithDanglingCcus inMem file g scope p x = p_memory (stringTab.AsArray, pubpathTab.AsArray, nlerefTab.AsArray, simpleTyTab.AsArray, phase1bytes) st2 - st2.os.AsMemory() + st2.os - let finalBytes = phase2bytes.ToArray() + let finalBytes = phase2bytes (st1.os :> System.IDisposable).Dispose() - (st2.os :> System.IDisposable).Dispose() finalBytes let check (ilscope: ILScopeRef) (inMap : NodeInTable<_, _>) = diff --git a/src/fsharp/TypedTreePickle.fsi b/src/fsharp/TypedTreePickle.fsi index 7a81358a003..5a2ade30289 100644 --- a/src/fsharp/TypedTreePickle.fsi +++ b/src/fsharp/TypedTreePickle.fsi @@ -84,7 +84,7 @@ val internal p_ty : pickler val internal pickleCcuInfo : pickler /// Serialize an arbitrary object using the given pickler -val pickleObjWithDanglingCcus : inMem: bool -> file: string -> TcGlobals -> scope:CcuThunk -> pickler<'T> -> 'T -> byte[] +val pickleObjWithDanglingCcus : inMem: bool -> file: string -> TcGlobals -> scope:CcuThunk -> pickler<'T> -> 'T -> ByteBuffer /// The type of state unpicklers read from type ReaderState diff --git a/src/fsharp/utils/FileSystem.fs b/src/fsharp/utils/FileSystem.fs index 3f7cb9520c0..7c575b8d44e 100644 --- a/src/fsharp/utils/FileSystem.fs +++ b/src/fsharp/utils/FileSystem.fs @@ -319,31 +319,45 @@ type ReadOnlyByteMemory(bytes: ByteMemory) = [] module MemoryMappedFileExtensions = - type MemoryMappedFile with - static member TryFromByteMemory(bytes: ReadOnlyByteMemory) = - let length = int64 bytes.Length - if length = 0L then + + let private trymmf length copyTo = + let length = int64 length + if length = 0L then + None + else + if runningOnMono then + // mono's MemoryMappedFile implementation throws with null `mapName`, so we use byte arrays instead: https://github.com/mono/mono/issues/1024 None else - if runningOnMono then - // mono's MemoryMappedFile implementation throws with null `mapName`, so we use byte arrays instead: https://github.com/mono/mono/issues/1024 - None - else - // Try to create a memory mapped file and copy the contents of the given bytes to it. - // If this fails, then we clean up and return None. + // Try to create a memory mapped file and copy the contents of the given bytes to it. + // If this fails, then we clean up and return None. + try + let mmf = MemoryMappedFile.CreateNew(null, length, MemoryMappedFileAccess.ReadWrite, MemoryMappedFileOptions.None, HandleInheritability.None) try - let mmf = MemoryMappedFile.CreateNew(null, length, MemoryMappedFileAccess.ReadWrite, MemoryMappedFileOptions.None, HandleInheritability.None) - try - use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite) - bytes.CopyTo stream - Some mmf - with - | _ -> - mmf.Dispose() - None + use stream = mmf.CreateViewStream(0L, length, MemoryMappedFileAccess.ReadWrite) + copyTo stream + Some mmf with | _ -> + mmf.Dispose() None + with + | _ -> + None + + type MemoryMappedFile with + static member TryFromByteMemory(bytes: ReadOnlyByteMemory) = + trymmf (int64 bytes.Length) bytes.CopyTo + + static member TryFromMemory(bytes: ReadOnlyMemory) = + let length = int64 bytes.Length + trymmf length + (fun stream -> + stream.SetLength(stream.Length + length) + let span = Span(stream.PositionPointer |> NativePtr.toVoidPtr, int length) + bytes.Span.CopyTo(span) + stream.Position <- stream.Position + length + ) [] module internal FileSystemUtils = @@ -926,5 +940,17 @@ type ByteStorage(getByteMemory: unit -> ReadOnlyByteMemory) = let copiedBytes = ByteMemory.FromArray(bytes.ToArray()).AsReadOnly() ByteStorage.FromByteMemory(copiedBytes) + static member FromMemoryAndCopy(bytes: ReadOnlyMemory, useBackingMemoryMappedFile: bool) = + if useBackingMemoryMappedFile then + match MemoryMappedFile.TryFromMemory(bytes) with + | Some mmf -> + ByteStorage(fun () -> ByteMemory.FromMemoryMappedFile(mmf).AsReadOnly()) + | _ -> + let copiedBytes = ByteMemory.FromArray(bytes.ToArray()).AsReadOnly() + ByteStorage.FromByteMemory(copiedBytes) + else + let copiedBytes = ByteMemory.FromArray(bytes.ToArray()).AsReadOnly() + ByteStorage.FromByteMemory(copiedBytes) + static member FromByteArrayAndCopy(bytes: byte [], useBackingMemoryMappedFile: bool) = ByteStorage.FromByteMemoryAndCopy(ByteMemory.FromArray(bytes).AsReadOnly(), useBackingMemoryMappedFile) diff --git a/src/fsharp/utils/FileSystem.fsi b/src/fsharp/utils/FileSystem.fsi index 616dcee939b..7a4236b0eee 100644 --- a/src/fsharp/utils/FileSystem.fsi +++ b/src/fsharp/utils/FileSystem.fsi @@ -97,6 +97,7 @@ type internal ReadOnlyByteMemory = module internal MemoryMappedFileExtensions = type MemoryMappedFile with static member TryFromByteMemory : bytes: ReadOnlyByteMemory -> MemoryMappedFile option + static member TryFromMemory : bytes: ReadOnlyMemory -> MemoryMappedFile option /// Filesystem helpers module internal FileSystemUtils = @@ -400,5 +401,8 @@ type internal ByteStorage = /// Creates a ByteStorage that has a copy of the given ByteMemory. static member FromByteMemoryAndCopy : ReadOnlyByteMemory * useBackingMemoryMappedFile: bool -> ByteStorage + /// Creates a ByteStorage that has a copy of the given Memory. + static member FromMemoryAndCopy : ReadOnlyMemory * useBackingMemoryMappedFile: bool -> ByteStorage + /// Creates a ByteStorage that has a copy of the given byte array. static member FromByteArrayAndCopy : byte [] * useBackingMemoryMappedFile: bool -> ByteStorage From f0452474a9c8dafe84cea3ef6cdd819a11211b5a Mon Sep 17 00:00:00 2001 From: Will Smith Date: Wed, 9 Jun 2021 14:51:50 -0700 Subject: [PATCH 06/16] Using ArrayPool in ilwrite --- src/fsharp/absil/ilwrite.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fsharp/absil/ilwrite.fs b/src/fsharp/absil/ilwrite.fs index e6ccc4b6f9f..c0ac3cecf13 100644 --- a/src/fsharp/absil/ilwrite.fs +++ b/src/fsharp/absil/ilwrite.fs @@ -3241,7 +3241,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca codedBigness 2 TableNames.AssemblyRef || codedBigness 2 TableNames.TypeRef - use tablesBuf = ByteBuffer.Create 20000 + use tablesBuf = ByteBuffer.Create(20000, useArrayPool = true) // Now the coded tables themselves - first the schemata header tablesBuf.EmitIntsAsBytes @@ -3320,7 +3320,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca reportTime showTimes "Layout Metadata" let metadata, guidStart = - use mdbuf = ByteBuffer.Create 500000 + use mdbuf = ByteBuffer.Create(500000, useArrayPool = true) mdbuf.EmitIntsAsBytes [| 0x42; 0x53; 0x4a; 0x42 // Magic signature 0x01; 0x00 // Major version From 97b04f31f570eddea5f4263d8695f0cf229383b7 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Wed, 9 Jun 2021 14:59:06 -0700 Subject: [PATCH 07/16] Fixing build --- src/fsharp/utils/FileSystem.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fsharp/utils/FileSystem.fs b/src/fsharp/utils/FileSystem.fs index 7c575b8d44e..b4e7b3c36dc 100644 --- a/src/fsharp/utils/FileSystem.fs +++ b/src/fsharp/utils/FileSystem.fs @@ -788,7 +788,7 @@ type internal ByteBuffer = if buf.useArrayPool then ArrayPool.Shared.Rent (max newSize (oldBufSize * 2)) else - Bytes.sub old 0 (max newSize (oldBufSize * 2)) + Bytes.zeroCreate (max newSize (oldBufSize * 2)) Bytes.blit old 0 buf.bbArray 0 buf.bbCurrent if buf.useArrayPool then ArrayPool.Shared.Return old From ae1707c90c5814b1396450de7aa524addb4e8420 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Wed, 9 Jun 2021 15:03:23 -0700 Subject: [PATCH 08/16] Fixed creation of byte buffer --- src/fsharp/utils/FileSystem.fs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/fsharp/utils/FileSystem.fs b/src/fsharp/utils/FileSystem.fs index b4e7b3c36dc..00b5e70a51b 100644 --- a/src/fsharp/utils/FileSystem.fs +++ b/src/fsharp/utils/FileSystem.fs @@ -891,9 +891,10 @@ type internal ByteBuffer = buf.bbCurrent static member Create(sz, useArrayPool) = - { useArrayPool = defaultArg useArrayPool false + let useArrayPool = defaultArg useArrayPool false + { useArrayPool = useArrayPool isDisposed = false - bbArray = ArrayPool.Shared.Rent sz + bbArray = if useArrayPool then ArrayPool.Shared.Rent sz else Bytes.zeroCreate sz bbCurrent = 0 } interface IDisposable with From 4bd3b9025950fdce2ec09e0d92357f3c634da9d7 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Mon, 14 Jun 2021 14:34:23 -0700 Subject: [PATCH 09/16] Remove EmitByteBuffer as it is not needed --- src/fsharp/utils/FileSystem.fs | 8 -------- src/fsharp/utils/FileSystem.fsi | 3 --- 2 files changed, 11 deletions(-) diff --git a/src/fsharp/utils/FileSystem.fs b/src/fsharp/utils/FileSystem.fs index 00b5e70a51b..bd453686287 100644 --- a/src/fsharp/utils/FileSystem.fs +++ b/src/fsharp/utils/FileSystem.fs @@ -857,14 +857,6 @@ type internal ByteBuffer = i.Copy(0, buf.bbArray, buf.bbCurrent, n) buf.bbCurrent <- newSize - member buf.EmitByteBuffer (i:ByteBuffer) = - buf.CheckDisposed() - let n = i.Position - let newSize = buf.bbCurrent + n - buf.Ensure newSize - Bytes.blit i.bbArray 0 buf.bbArray buf.bbCurrent n - buf.bbCurrent <- newSize - member buf.EmitInt32AsUInt16 n = buf.CheckDisposed() let newSize = buf.bbCurrent + 2 diff --git a/src/fsharp/utils/FileSystem.fsi b/src/fsharp/utils/FileSystem.fsi index 7a4236b0eee..212d13ef1d0 100644 --- a/src/fsharp/utils/FileSystem.fsi +++ b/src/fsharp/utils/FileSystem.fsi @@ -363,9 +363,6 @@ type internal ByteBuffer = [] member EmitByteMemory : ReadOnlyByteMemory -> unit - [] - member EmitByteBuffer : ByteBuffer -> unit - [] member EmitInt32 : int32 -> unit From 2fc33a8fd031fcd2ca798815174365dd0c965b9a Mon Sep 17 00:00:00 2001 From: Will Smith Date: Mon, 14 Jun 2021 15:44:38 -0700 Subject: [PATCH 10/16] Added System.Runtime.CompilerServices.Unsafe to be version 4.7.1 --- eng/Versions.props | 1 + .../FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj | 2 ++ .../FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec | 1 + vsintegration/Directory.Build.targets | 1 + 4 files changed, 5 insertions(+) diff --git a/eng/Versions.props b/eng/Versions.props index dfab66e55a9..62b2ac6a40c 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -75,6 +75,7 @@ https://pkgs.dev.azure.com/azure-public/vside/_packaging/vs-impl/nuget/v3/index.json; + 4.7.1 4.5.1 5.0.0 4.3.0 diff --git a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj index cfbae17b9a1..8a63612ce3d 100644 --- a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj +++ b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj @@ -39,6 +39,7 @@ + @@ -983,6 +984,7 @@ + diff --git a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec index c7f35f65030..94d1a5722b9 100644 --- a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec +++ b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec @@ -10,6 +10,7 @@ + diff --git a/vsintegration/Directory.Build.targets b/vsintegration/Directory.Build.targets index d47467d76c4..170ace14822 100644 --- a/vsintegration/Directory.Build.targets +++ b/vsintegration/Directory.Build.targets @@ -14,6 +14,7 @@ + From 7a57ef949e51f780707f6abaf5b31e8c6292cfd7 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Tue, 15 Jun 2021 11:47:58 -0700 Subject: [PATCH 11/16] Update Versions.props --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 62b2ac6a40c..1fa6e0bb15e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -75,7 +75,6 @@ https://pkgs.dev.azure.com/azure-public/vside/_packaging/vs-impl/nuget/v3/index.json; - 4.7.1 4.5.1 5.0.0 4.3.0 @@ -104,6 +103,7 @@ 4.11.1 4.3.0 4.3.0 + 4.7.1 3.8.0-5.20570.14 $(RoslynVersion) From 7d6db98464e0ea396341b93dc7605813c5e8f67f Mon Sep 17 00:00:00 2001 From: Will Smith Date: Tue, 15 Jun 2021 12:23:52 -0700 Subject: [PATCH 12/16] Update FSharp.Compiler.Service.fsproj --- .../FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj index 8a63612ce3d..dd96b7da59e 100644 --- a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj +++ b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.fsproj @@ -39,7 +39,6 @@ - @@ -60,6 +59,7 @@ + @@ -984,11 +984,11 @@ - + From a86bc9cff6bcfae831dc854e954151775af9b33d Mon Sep 17 00:00:00 2001 From: Will Smith Date: Tue, 15 Jun 2021 12:24:42 -0700 Subject: [PATCH 13/16] Update FSharp.Compiler.Service.nuspec --- .../FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec index 94d1a5722b9..065f1cdb31c 100644 --- a/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec +++ b/src/fsharp/FSharp.Compiler.Service/FSharp.Compiler.Service.nuspec @@ -10,7 +10,6 @@ - @@ -31,6 +30,7 @@ + From 77f83b8ab50448167a8d7f78747b1fb642e165d9 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Tue, 15 Jun 2021 14:14:05 -0700 Subject: [PATCH 14/16] Added literals for byte buffer creation --- src/fsharp/QuotationPickler.fs | 4 +++ src/fsharp/TypedTreePickle.fs | 8 ++++-- src/fsharp/absil/ilwrite.fs | 38 +++++++++++++++++++++++------ src/fsharp/lex.fsl | 8 ++++-- src/fsharp/service/ServiceLexing.fs | 2 +- src/fsharp/utils/FileSystem.fs | 4 +-- src/fsharp/utils/FileSystem.fsi | 2 +- 7 files changed, 51 insertions(+), 15 deletions(-) diff --git a/src/fsharp/QuotationPickler.fs b/src/fsharp/QuotationPickler.fs index 5e0abd1052a..869c259ad43 100644 --- a/src/fsharp/QuotationPickler.fs +++ b/src/fsharp/QuotationPickler.fs @@ -246,6 +246,10 @@ let SerializedReflectedDefinitionsResourceNameBase = "ReflectedDefinitions" let freshVar (n, ty, mut) = { vText=n; vType=ty; vMutable=mut } +/// Arbitrary value +[] +let PickleBufferCapacity = 100000 + module SimplePickle = type Table<'T> = diff --git a/src/fsharp/TypedTreePickle.fs b/src/fsharp/TypedTreePickle.fs index 21aefc62497..b8325c261d1 100644 --- a/src/fsharp/TypedTreePickle.fs +++ b/src/fsharp/TypedTreePickle.fs @@ -780,9 +780,13 @@ let p_encoded_simpletyp x st = p_int x st let p_encoded_anoninfo x st = p_int x st let p_simpletyp x st = p_int (encode_simpletyp st.occus st.ostrings st.onlerefs st.osimpletys st.oscope x) st +/// Arbitrary value +[] +let PickleBufferCapacity = 100000 + let pickleObjWithDanglingCcus inMem file g scope p x = let st1 = - { os = ByteBuffer.Create(100000, useArrayPool = true) + { os = ByteBuffer.Create(PickleBufferCapacity, useArrayPool = true) oscope=scope occus= Table<_>.Create "occus" oentities=NodeOutTable<_, _>.Create((fun (tc: Tycon) -> tc.Stamp), (fun tc -> tc.LogicalName), (fun tc -> tc.Range), (fun osgn -> osgn), "otycons") @@ -807,7 +811,7 @@ let pickleObjWithDanglingCcus inMem file g scope p x = st1.occus, sizes, st1.ostrings, st1.opubpaths, st1.onlerefs, st1.osimpletys, st1.os.AsMemory() let st2 = - { os = ByteBuffer.Create(100000, useArrayPool = true) + { os = ByteBuffer.Create(PickleBufferCapacity, useArrayPool = true) oscope=scope occus= Table<_>.Create "occus (fake)" oentities=NodeOutTable<_, _>.Create((fun (tc: Tycon) -> tc.Stamp), (fun tc -> tc.LogicalName), (fun tc -> tc.Range), (fun osgn -> osgn), "otycons") diff --git a/src/fsharp/absil/ilwrite.fs b/src/fsharp/absil/ilwrite.fs index c0ac3cecf13..2abd677d08e 100644 --- a/src/fsharp/absil/ilwrite.fs +++ b/src/fsharp/absil/ilwrite.fs @@ -48,7 +48,10 @@ let dw0 n = byte (n &&& 0xFFL) let bitsOfSingle (x: float32) = System.BitConverter.ToInt32(System.BitConverter.GetBytes x, 0) let bitsOfDouble (x: float) = System.BitConverter.DoubleToInt64Bits x -let emitBytesViaBuffer f = use bb = ByteBuffer.Create 10 in f bb; bb.AsMemory().ToArray() +/// Arbitrary value +[] +let EmitBytesViaBufferCapacity = 10 +let emitBytesViaBuffer f = use bb = ByteBuffer.Create EmitBytesViaBufferCapacity in f bb; bb.AsMemory().ToArray() /// Alignment and padding let align alignment n = ((n + alignment - 1) / alignment) * alignment @@ -1453,6 +1456,10 @@ type ExceptionClauseKind = type ExceptionClauseSpec = (int * int * int * int * ExceptionClauseKind) +/// Arbitrary value +[] +let CodeBufferCapacity = 200 + type CodeBuffer = // -------------------------------------------------------------------- @@ -1478,7 +1485,7 @@ type CodeBuffer = static member Create _nm = { seh = [] - code= ByteBuffer.Create 200 + code= ByteBuffer.Create CodeBufferCapacity reqdBrFixups=[] reqdStringFixupsInMethod=[] availBrFixups = Dictionary<_, _>(10, HashIdentity.Structural) @@ -2870,6 +2877,16 @@ let GenModule (cenv : cenv) (modul: ILModuleDef) = GenTypeDefsPass4 [] cenv tds reportTime cenv.showTimes "Module Generation Pass 4" +/// Arbitrary value +[] +let CodeChunkCapacity = 40000 +/// Arbitrary value +[] +let DataCapacity = 200 +/// Arbitrary value +[] +let ResourceCapacity = 200 + let generateIL requiredDataFixups (desiredMetadataVersion, generatePdb, ilg : ILGlobals, emitTailcalls, deterministic, showTimes) (m : ILModuleDef) cilStartAddress normalizeAssemblyRefs = let isDll = m.IsDLL @@ -2881,10 +2898,10 @@ let generateIL requiredDataFixups (desiredMetadataVersion, generatePdb, ilg : IL desiredMetadataVersion=desiredMetadataVersion requiredDataFixups= requiredDataFixups requiredStringFixups = [] - codeChunks=ByteBuffer.Create(40000, useArrayPool = true) + codeChunks=ByteBuffer.Create(CodeChunkCapacity, useArrayPool = true) nextCodeAddr = cilStartAddress - data = ByteBuffer.Create 200 - resources = ByteBuffer.Create 200 + data = ByteBuffer.Create DataCapacity + resources = ByteBuffer.Create ResourceCapacity tables= Array.init 64 (fun i -> if (i = TableNames.AssemblyRef.Index || @@ -3014,6 +3031,13 @@ module FileSystemUtilities = #endif () +/// Arbitrary value +[] +let TableCapacity = 20000 +/// Arbitrary value +[] +let MetadataCapacity = 500000 + let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailcalls, deterministic, showTimes) modul cilStartAddress normalizeAssemblyRefs = // When we know the real RVAs of the data section we fixup the references for the FieldRVA table. @@ -3241,7 +3265,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca codedBigness 2 TableNames.AssemblyRef || codedBigness 2 TableNames.TypeRef - use tablesBuf = ByteBuffer.Create(20000, useArrayPool = true) + use tablesBuf = ByteBuffer.Create(TableCapacity, useArrayPool = true) // Now the coded tables themselves - first the schemata header tablesBuf.EmitIntsAsBytes @@ -3320,7 +3344,7 @@ let writeILMetadataAndCode (generatePdb, desiredMetadataVersion, ilg, emitTailca reportTime showTimes "Layout Metadata" let metadata, guidStart = - use mdbuf = ByteBuffer.Create(500000, useArrayPool = true) + use mdbuf = ByteBuffer.Create(MetadataCapacity, useArrayPool = true) mdbuf.EmitIntsAsBytes [| 0x42; 0x53; 0x4a; 0x42 // Magic signature 0x01; 0x00 // Major version diff --git a/src/fsharp/lex.fsl b/src/fsharp/lex.fsl index 0aa3bc11ce3..40da3928359 100644 --- a/src/fsharp/lex.fsl +++ b/src/fsharp/lex.fsl @@ -130,8 +130,12 @@ let checkExprOp (lexbuf:UnicodeLexing.Lexbuf) = let unexpectedChar lexbuf = LEX_FAILURE (FSComp.SR.lexUnexpectedChar(lexeme lexbuf)) +/// Arbitrary value +[] +let StringCapacity = 100 + let startString args (lexbuf: UnicodeLexing.Lexbuf) = - let buf = ByteBuffer.Create 100 + let buf = ByteBuffer.Create StringCapacity let m = lexbuf.LexemeRange let startp = lexbuf.StartPos let fin = @@ -1584,7 +1588,7 @@ and tripleQuoteStringInComment n m args skip = parse and mlOnly m args skip = parse | "\"" - { let buf = ByteBuffer.Create 100 + { let buf = ByteBuffer.Create StringCapacity let m2 = lexbuf.LexemeRange let _ = singleQuoteString (buf, LexerStringFinisher.Default, m2, LexerStringKind.String, args) skip lexbuf if not skip then COMMENT (LexCont.MLOnly(args.ifdefStack, args.stringNest, m)) diff --git a/src/fsharp/service/ServiceLexing.fs b/src/fsharp/service/ServiceLexing.fs index 1de7f74607a..b0b66a6f35d 100644 --- a/src/fsharp/service/ServiceLexing.fs +++ b/src/fsharp/service/ServiceLexing.fs @@ -706,7 +706,7 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, | LexCont.String (ifdefs, stringNest, style, kind, m) -> lexargs.ifdefStack <- ifdefs lexargs.stringNest <- stringNest - use buf = ByteBuffer.Create 100 + use buf = ByteBuffer.Create (Lexer.StringCapacity) let args = (buf, LexerStringFinisher.Default, m, kind, lexargs) match style with | LexerStringStyle.SingleQuote -> Lexer.singleQuoteString args skip lexbuf diff --git a/src/fsharp/utils/FileSystem.fs b/src/fsharp/utils/FileSystem.fs index bd453686287..9346e6ab1a8 100644 --- a/src/fsharp/utils/FileSystem.fs +++ b/src/fsharp/utils/FileSystem.fs @@ -882,11 +882,11 @@ type internal ByteBuffer = buf.CheckDisposed() buf.bbCurrent - static member Create(sz, useArrayPool) = + static member Create(capacity, useArrayPool) = let useArrayPool = defaultArg useArrayPool false { useArrayPool = useArrayPool isDisposed = false - bbArray = if useArrayPool then ArrayPool.Shared.Rent sz else Bytes.zeroCreate sz + bbArray = if useArrayPool then ArrayPool.Shared.Rent capacity else Bytes.zeroCreate capacity bbCurrent = 0 } interface IDisposable with diff --git a/src/fsharp/utils/FileSystem.fsi b/src/fsharp/utils/FileSystem.fsi index 212d13ef1d0..14bc45a9850 100644 --- a/src/fsharp/utils/FileSystem.fsi +++ b/src/fsharp/utils/FileSystem.fsi @@ -382,7 +382,7 @@ type internal ByteBuffer = member EmitUInt16 : uint16 -> unit member Position : int - static member Create : int * ?useArrayPool: bool -> ByteBuffer + static member Create : capacity: int * ?useArrayPool: bool -> ByteBuffer [] type internal ByteStorage = From b49977a3c47f41680c99184daa8c062877e76d49 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Tue, 15 Jun 2021 14:18:29 -0700 Subject: [PATCH 15/16] minor tweak --- src/fsharp/QuotationPickler.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fsharp/QuotationPickler.fs b/src/fsharp/QuotationPickler.fs index 869c259ad43..abcf1a42882 100644 --- a/src/fsharp/QuotationPickler.fs +++ b/src/fsharp/QuotationPickler.fs @@ -374,7 +374,7 @@ module SimplePickle = let pickle_obj p x = let st1 = - { os = ByteBuffer.Create(100000, useArrayPool = true) + { os = ByteBuffer.Create(PickleBufferCapacity, useArrayPool = true) ostrings=Table<_>.Create() } let stringTab, phase1bytes = p x st1 @@ -383,7 +383,7 @@ module SimplePickle = let phase2data = (stringTab, phase1bytes) let st2 = - { os = ByteBuffer.Create(100000, useArrayPool = true) + { os = ByteBuffer.Create(PickleBufferCapacity, useArrayPool = true) ostrings=Table<_>.Create() } let phase2bytes = p_tup2 (p_list prim_pstring) p_memory phase2data st2 From 4bc078252d1df3fefe5388f24675bc122a59eb88 Mon Sep 17 00:00:00 2001 From: Will Smith Date: Tue, 15 Jun 2021 16:38:12 -0700 Subject: [PATCH 16/16] Update TypedTree.fs --- src/fsharp/TypedTree.fs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/fsharp/TypedTree.fs b/src/fsharp/TypedTree.fs index c828f46aeb6..aa1ae4ee748 100644 --- a/src/fsharp/TypedTree.fs +++ b/src/fsharp/TypedTree.fs @@ -26,7 +26,6 @@ open FSharp.Compiler.SyntaxTreeOps open FSharp.Compiler.Text open FSharp.Compiler.Text.Range open FSharp.Compiler.Xml -open FSharp.Compiler.IO #if !NO_EXTENSIONTYPING open FSharp.Compiler.ExtensionTyping