Embedded source

build.cmd

@@ -64,6 +64,8 @@ for %%i in (%BUILD_FSC_DEFAULT%) do ( call :SET_CONFIG %%i )
 setlocal disableDelayedExpansion
 echo.
 
+rem disable setup build by setting FSC_BUILD_SETUP=0
+if /i '%FSC_BUILD_SETUP%' == '' (set FSC_BUILD_SETUP=1) 
 goto :MAIN
 
 :SET_CONFIG
@@ -96,7 +98,7 @@ if /i '%ARG%' == 'all' (
     set BUILD_CORECLR=1
     set BUILD_PORTABLE=1
     set BUILD_VS=1
-    set BUILD_SETUP=1
+    set BUILD_SETUP=%FSC_BUILD_SETUP%
 
     set TEST_COMPILERUNIT=1
     set TEST_NET40_COREUNIT=1
@@ -120,7 +122,7 @@ if /i '%ARG%' == 'microbuild' (
     set BUILD_CORECLR=0
     set BUILD_PORTABLE=1
     set BUILD_VS=1
-    set BUILD_SETUP=1
+    set BUILD_SETUP=%FSC_BUILD_SETUP%
 
     set TEST_COMPILERUNIT=1
     set TEST_NET40_COREUNIT=1
@@ -143,7 +145,7 @@ if /i '%ARG%' == 'ci' (
     set BUILD_CORECLR=1
     set BUILD_PORTABLE=1
     set BUILD_VS=1
-    set BUILD_SETUP=1
+    set BUILD_SETUP=%FSC_BUILD_SETUP%
 
     set TEST_COMPILERUNIT=1
     set TEST_NET40_COREUNIT=1
@@ -164,7 +166,7 @@ if /i '%ARG%' == 'ci_part1' (
     set BUILD_CORECLR=0
     set BUILD_PORTABLE=1
     set BUILD_VS=1
-    set BUILD_SETUP=1
+    set BUILD_SETUP=%FSC_BUILD_SETUP%
 
     set TEST_COMPILERUNIT=1
     set TEST_NET40_COREUNIT=0

src/FSharpSource.Settings.targets

@@ -26,10 +26,11 @@
   <!-- Standard interpretations of Debug and Release configurations -->
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
     <DebugType Condition=" '$(DebugType)' == '' and  '$(TargetFramework)' != 'coreclr' ">full</DebugType>
-    <DebugType Condition=" '$(DebugType)' == '' and  '$(TargetFramework)' == 'coreclr' ">portable</DebugType>
+    <DebugType Condition=" '$(DebugType)' == '' and  '$(TargetFramework)' == 'coreclr' ">embedded</DebugType>
     <Optimize Condition=" '$(Optimize)' == '' ">false</Optimize>
     <ErrorReport Condition=" '$(ErrorReport)' == '' ">prompt</ErrorReport>
-    <OtherFlags>$(OtherFlags) --no-jit-optimize</OtherFlags>
+    <OtherFlags Condition=" '$(DebugType)' == '' and  '$(TargetFramework)' != 'coreclr' ">$(OtherFlags) --no-jit-optimize</OtherFlags>
+    <OtherFlags Condition=" '$(DebugType)' == '' and  '$(TargetFramework)' == 'coreclr' ">$(OtherFlags) --no-jit-optimize --embed</OtherFlags>
     <DefineConstants Condition=" '$(ProjectLanguage)' != 'VisualBasic' ">DEBUG;TRACE;CODE_ANALYSIS;$(DefineConstants)</DefineConstants>
     <DefineConstants Condition=" '$(ProjectLanguage)' == 'VisualBasic' ">DEBUG=True,TRACE=True,CODE_ANALYSIS=True,$(DefineConstants)</DefineConstants>
     <SIGN_WITH_MSFT_KEY Condition=" '$(SIGN_WITH_MSFT_KEY)' == '' ">false</SIGN_WITH_MSFT_KEY>

src/absil/ilwrite.fs

@@ -3536,7 +3536,7 @@ let writeDirectory os dict =
 
 let writeBytes (os: BinaryWriter) (chunk:byte[]) = os.Write(chunk,0,chunk.Length)  
 
-let writeBinaryAndReportMappings (outfile, ilg, pdbfile: string option, signer: ILStrongNameSigner option, portablePDB, embeddedPDB, 
+let writeBinaryAndReportMappings (outfile, ilg, pdbfile: string option, signer: ILStrongNameSigner option, portablePDB, embeddedPDB, embedAllSource, embedSourceList,
                                   fixupOverlappingSequencePoints, emitTailcalls, showTimes, dumpDebugInfo) modul noDebugData =
     // Store the public key from the signer into the manifest.  This means it will be written 
     // to the binary and also acts as an indicator to leave space for delay sign 
@@ -3690,7 +3690,7 @@ let writeBinaryAndReportMappings (outfile, ilg, pdbfile: string option, signer:
           let pdbOpt =
             match portablePDB with
             | true  -> 
-                let struct (uncompressedLength, contentId, stream) as pdbStream = generatePortablePdb fixupOverlappingSequencePoints showTimes pdbData
+                let struct (uncompressedLength, contentId, stream) as pdbStream = generatePortablePdb fixupOverlappingSequencePoints embedAllSource embedSourceList showTimes pdbData 
                 if embeddedPDB then Some (compressPortablePdbStream uncompressedLength contentId stream)
                 else Some (pdbStream)
             | _ -> None
@@ -4260,13 +4260,15 @@ type options =
      pdbfile: string option
      portablePDB: bool
      embeddedPDB: bool
+     embedAllSource: bool
+     embedSourceList: string list
      signer: ILStrongNameSigner option
      fixupOverlappingSequencePoints: bool
      emitTailcalls : bool
      showTimes: bool
      dumpDebugInfo:bool }
 
 let WriteILBinary (outfile, (args: options), modul, noDebugData) =
-    ignore (writeBinaryAndReportMappings (outfile, args.ilg, args.pdbfile, args.signer, args.portablePDB, args.embeddedPDB,
-                                          args.fixupOverlappingSequencePoints, args.emitTailcalls, args.showTimes, 
-                                          args.dumpDebugInfo) modul noDebugData)
+    ignore (writeBinaryAndReportMappings (outfile, args.ilg, args.pdbfile, args.signer, args.portablePDB, args.embeddedPDB, 
+                                          args.embedAllSource, args.embedSourceList, args.fixupOverlappingSequencePoints, 
+                                          args.emitTailcalls, args.showTimes, args.dumpDebugInfo) modul noDebugData)

src/absil/ilwrite.fsi

@@ -20,6 +20,8 @@ type options =
    pdbfile: string option
    portablePDB: bool
    embeddedPDB: bool
+   embedAllSource: bool
+   embedSourceList: string list
    signer : ILStrongNameSigner option
    fixupOverlappingSequencePoints : bool
    emitTailcalls: bool

src/absil/ilwritepdb.fs

@@ -19,6 +19,31 @@ open Microsoft.FSharp.Compiler.AbstractIL.Internal.Library
 open Microsoft.FSharp.Compiler.ErrorLogger
 open Microsoft.FSharp.Compiler.Range
 
+
+type BlobBuildingStream () =
+    inherit Stream()
+
+    static let chunkSize = 32 * 1024
+    let builder = new BlobBuilder(chunkSize)
+
+    override this.CanWrite  with get() = true
+    override this.CanRead   with get() = false
+    override this.CanSeek   with get() = false
+    override this.Length    with get() = int64(builder.Count)
+
+    override this.Write(buffer:byte array, offset:int, count:int) = builder.WriteBytes(buffer, offset, count)
+    override this.WriteByte(value:byte) = builder.WriteByte(value)
+    member   this.WriteInt32(value:int) = builder.WriteInt32(value)
+    member   this.ToImmutableArray() = builder.ToImmutableArray()
+    member   this.TryWriteBytes(stream:Stream, length:int) = builder.TryWriteBytes(stream, length)
+
+    override this.Flush() = ()
+    override this.Dispose(_disposing:bool) = ()
+    override this.Seek(_offset:int64, _origin:SeekOrigin) = raise (new NotSupportedException())
+    override this.Read(_buffer:byte array, _offset:int, _count:int) = raise (new NotSupportedException())
+    override this.SetLength(_value:int64) = raise (new NotSupportedException())
+    override val Position = 0L with get, set
+
 // -------------------------------------------------------------------- 
 // PDB types
 // --------------------------------------------------------------------  
@@ -227,7 +252,7 @@ let fixupOverlappingSequencePoints fixupSPs showTimes methods =
         Array.sortInPlaceBy fst allSps
     spCounts, allSps
 
-let generatePortablePdb fixupSPs showTimes (info:PdbData) = 
+let generatePortablePdb fixupSPs (embedAllSource:bool) (embedSourceList:string list) showTimes (info:PdbData) = 
     sortMethods showTimes info
     let _spCounts, _allSps = fixupOverlappingSequencePoints fixupSPs showTimes info.Methods
     let externalRowCounts = getRowCounts info.TableRowCounts
@@ -253,23 +278,71 @@ let generatePortablePdb fixupSPs showTimes (info:PdbData) =
         metadata.GetOrAddBlob(writer)
 
     let corSymLanguageTypeFSharp = System.Guid(0xAB4F38C9u, 0xB6E6us, 0x43baus, 0xBEuy, 0x3Buy, 0x58uy, 0x08uy, 0x0Buy, 0x2Cuy, 0xCCuy, 0xE3uy)
+    let embeddedSource = System.Guid(0x0e8a571bu, 0x6926us, 0x466eus, 0xb4uy, 0xaduy, 0x8auy, 0xb0uy, 0x46uy, 0x11uy, 0xf5uy, 0xfeuy)
+
+    /// <summary>
+    /// The maximum number of bytes in to write out uncompressed.
+    ///
+    /// This prevents wasting resources on compressing tiny files with little to negative gain
+    /// in PDB file size.
+    ///
+    /// Chosen as the point at which we start to see > 10% blob size reduction using all
+    /// current source files in corefx and roslyn as sample data. 
+    /// </summary>
+    let sourceCompressionThreshold = 200
+
     let documentIndex =
+        let includeSource file =
+            let isInList =
+                if embedSourceList |> List.length = 0 then false
+                else
+                    match embedSourceList |> List.tryFind(fun f -> String.Compare(file, f, StringComparison.OrdinalIgnoreCase ) = 0) with
+                    | Some _ -> true
+                    | None   -> false
+
+            if not embedAllSource && not isInList || not (File.Exists(file)) then
+                None
+            else
+                let stream = File.OpenRead(file)
+                let length64 = stream.Length
+                if length64 > int64(Int32.MaxValue) then raise (new IOException("File is too long"))
+
+                let builder = new BlobBuildingStream()
+                let length = int(length64)
+                if length < sourceCompressionThreshold then
+                    builder.WriteInt32(0)
+                    builder.TryWriteBytes(stream, length) |> ignore
+                else
+                    builder.WriteInt32(length) |>ignore
+                    use deflater = new DeflateStream(builder, CompressionMode.Compress, true)
+                    stream.CopyTo(deflater) |> ignore
+                Some (builder.ToImmutableArray())
+
         let mutable index = new Dictionary<string, DocumentHandle>(docs.Length)
         metadata.SetCapacity(TableIndex.Document, docs.Length)
         for doc in docs do
             let handle =
                 match checkSum doc.File with
                 | Some (hashAlg, checkSum) ->
-                    serializeDocumentName doc.File,
-                    metadata.GetOrAddGuid(hashAlg),
-                    metadata.GetOrAddBlob(checkSum.ToImmutableArray()),
-                    metadata.GetOrAddGuid(corSymLanguageTypeFSharp)
+                    let h = 
+                        (serializeDocumentName doc.File,
+                         metadata.GetOrAddGuid(hashAlg),
+                         metadata.GetOrAddBlob(checkSum.ToImmutableArray()),
+                         metadata.GetOrAddGuid(corSymLanguageTypeFSharp)) |> metadata.AddDocument
+                    match includeSource doc.File with
+                    | None -> ()
+                    | Some blob ->
+                        metadata.AddCustomDebugInformation(DocumentHandle.op_Implicit(h),
+                                                           metadata.GetOrAddGuid(embeddedSource),
+                                                           metadata.GetOrAddBlob(blob)) |> ignore
+                    h
                 | None ->
-                    serializeDocumentName doc.File,
-                    metadata.GetOrAddGuid(System.Guid.Empty),
-                    metadata.GetOrAddBlob(ImmutableArray<byte>.Empty),
-                    metadata.GetOrAddGuid(corSymLanguageTypeFSharp)
-                |> metadata.AddDocument
+                    let h = 
+                        (serializeDocumentName doc.File,
+                         metadata.GetOrAddGuid(System.Guid.Empty),
+                         metadata.GetOrAddBlob(ImmutableArray<byte>.Empty),
+                         metadata.GetOrAddGuid(corSymLanguageTypeFSharp)) |> metadata.AddDocument
+                    h
             index.Add(doc.File, handle)
         index
 
@@ -291,7 +364,7 @@ let generatePortablePdb fixupSPs showTimes (info:PdbData) =
                 else 
                     match documentIndex.TryGetValue(docs.[d].File) with
                     | false, _ -> Unchecked.defaultof<DocumentHandle>
-                    | true, f  -> f
+                    | true, h -> h
 
             if sps.Length = 0 then
                 Unchecked.defaultof<DocumentHandle>, Unchecked.defaultof<BlobHandle>
@@ -306,7 +379,6 @@ let generatePortablePdb fixupSPs showTimes (info:PdbData) =
                     singleDocumentIndex
 
                 let builder = new BlobBuilder()
-
                 builder.WriteCompressedInteger(minfo.LocalSignatureToken)
 
                 // Initial document:  When sp's spread over more than one document we put the initial document here.

src/absil/ilwritepdb.fsi

@@ -82,7 +82,7 @@ type idd =
       iddData: byte[];
       iddChunk: BinaryChunk }
 
-val generatePortablePdb : fixupSPs:bool -> showTimes:bool -> info:PdbData -> struct (int64 * BlobContentId * MemoryStream) 
+val generatePortablePdb : fixupSPs:bool -> embedAllSource:bool -> embedSourceList:string list -> showTimes:bool -> info:PdbData -> struct (int64 * BlobContentId * MemoryStream) 
 val compressPortablePdbStream : uncompressedLength:int64 -> contentId:BlobContentId -> stream:MemoryStream -> struct (int64 * BlobContentId * MemoryStream)
 val embedPortablePdbInfo : uncompressedLength:int64 -> contentId:BlobContentId -> stream:MemoryStream -> showTimes:bool -> fpdb:string -> cvChunk:BinaryChunk -> pdbChunk:BinaryChunk -> idd[]
 val writePortablePdbInfo : contentId:BlobContentId -> stream:MemoryStream -> showTimes:bool -> fpdb:string -> cvChunk:BinaryChunk -> idd[]

src/fsharp/CompileOps.fs

@@ -2073,6 +2073,9 @@ type TcConfigBuilder =
       mutable jitTracking : bool
       mutable portablePDB : bool
       mutable embeddedPDB : bool
+      mutable embedAllSource : bool
+      mutable embedSourceList : string list 
+
       mutable ignoreSymbolStoreSequencePoints : bool
       mutable internConstantStrings : bool
       mutable extraOptimizationIterations : int
@@ -2242,7 +2245,9 @@ type TcConfigBuilder =
           useSignatureDataFile = false
           jitTracking = true
           portablePDB = true
-          embeddedPDB = true
+          embeddedPDB = false
+          embedAllSource = false
+          embedSourceList = []
           ignoreSymbolStoreSequencePoints = false
           internConstantStrings = true
           extraOptimizationIterations = 0
@@ -2371,7 +2376,7 @@ type TcConfigBuilder =
             | None -> false
         if ok && not (List.contains absolutePath tcConfigB.includes) then 
            tcConfigB.includes <- tcConfigB.includes ++ absolutePath
-           
+
     member tcConfigB.AddLoadedSource(m,path,pathLoadedFrom) =
         if FileSystem.IsInvalidPathShim(path) then
             warning(Error(FSComp.SR.buildInvalidFilename(path),m))    
@@ -2384,7 +2389,9 @@ type TcConfigBuilder =
                     ComputeMakePathAbsolute pathLoadedFrom path
             if not (List.contains path (List.map snd tcConfigB.loadedSources)) then 
                 tcConfigB.loadedSources <- tcConfigB.loadedSources ++ (m,path)
-
+
+    member tcConfigB.AddEmbeddedSourceFile (file) = 
+        tcConfigB.embedSourceList <- tcConfigB.embedSourceList ++ file
 
     member tcConfigB.AddEmbeddedResource filename =
         tcConfigB.embedResources <- tcConfigB.embedResources ++ filename
@@ -2732,6 +2739,8 @@ type TcConfig private (data : TcConfigBuilder,validate:bool) =
     member x.jitTracking  = data.jitTracking
     member x.portablePDB  = data.portablePDB
     member x.embeddedPDB  = data.embeddedPDB
+    member x.embedAllSource  = data.embedAllSource
+    member x.embedSourceList  = data.embedSourceList
     member x.ignoreSymbolStoreSequencePoints  = data.ignoreSymbolStoreSequencePoints
     member x.internConstantStrings  = data.internConstantStrings
     member x.extraOptimizationIterations  = data.extraOptimizationIterations

src/fsharp/CompileOps.fsi

@@ -307,6 +307,8 @@ type TcConfigBuilder =
       mutable jitTracking : bool
       mutable portablePDB : bool
       mutable embeddedPDB : bool
+      mutable embedAllSource : bool
+      mutable embedSourceList : string list
       mutable ignoreSymbolStoreSequencePoints : bool
       mutable internConstantStrings : bool
       mutable extraOptimizationIterations : int
@@ -377,6 +379,7 @@ type TcConfigBuilder =
     member AddIncludePath : range * string * string -> unit
     member AddReferencedAssemblyByPath : range * string -> unit
     member RemoveReferencedAssemblyByPath : range * string -> unit
+    member AddEmbeddedSourceFile : string -> unit
     member AddEmbeddedResource : string -> unit
 
     static member SplitCommandLineResourceInfo : string -> string * string * ILResourceAccess
@@ -461,6 +464,8 @@ type TcConfig =
     member jitTracking : bool
     member portablePDB : bool
     member embeddedPDB : bool
+    member embedAllSource : bool
+    member embedSourceList : string list
     member ignoreSymbolStoreSequencePoints : bool
     member internConstantStrings : bool
     member extraOptimizationIterations : int