Transparent Compiler various fixes

docs/release-notes/.FSharp.Compiler.Service/8.0.300.md

@@ -2,7 +2,7 @@
 
 * Code generated files with > 64K methods and generated symbols crash when loaded. Use infered sequence points for debugging. ([Issue #16399](https://github.com/dotnet/fsharp/issues/16399), [#PR 16514](https://github.com/dotnet/fsharp/pull/16514))
 * `nameof Module` expressions and patterns are processed to link files in `--test:GraphBasedChecking`. ([PR #16550](https://github.com/dotnet/fsharp/pull/16550))
-* Graph Based Checking doesn't throw on invalid parsed input so it can be used for IDE scenarios ([PR #16575](https://github.com/dotnet/fsharp/pull/16575), [PR #16588](https://github.com/dotnet/fsharp/pull/16588))
+* Graph Based Checking doesn't throw on invalid parsed input so it can be used for IDE scenarios ([PR #16575](https://github.com/dotnet/fsharp/pull/16575), [PR #16588](https://github.com/dotnet/fsharp/pull/16588), [PR #16643](https://github.com/dotnet/fsharp/pull/16643))
 * Keep parens for problematic exprs (`if`, `match`, etc.) in `$"{(…):N0}"`, `$"{(…),-3}"`, etc. ([PR #16578](https://github.com/dotnet/fsharp/pull/16578))
 * Fix crash in DOTNET_SYSTEM_GLOBALIZATION_INVARIANT mode [#PR 16471](https://github.com/dotnet/fsharp/pull/16471))
 

src/Compiler/Driver/GraphChecking/TrieMapping.fs

@@ -123,7 +123,7 @@ let processSynModuleOrNamespace<'Decl>
         // Only the last node can be a module, depending on the SynModuleOrNamespaceKind.
         let rec visit continuation (xs: LongIdent) =
             match xs with
-            | [] -> failwith "should not be empty"
+            | [] -> ImmutableDictionary.Empty |> continuation
             | [ finalPart ] ->
                 let name = finalPart.idText
 

src/Compiler/Service/FSharpProjectSnapshot.fs

@@ -364,7 +364,10 @@ and internal ProjectCore
              |> Md5Hasher.addDateTimes (ReferencesOnDisk |> Seq.map (fun r -> r.LastModified))
              |> Md5Hasher.addBytes' (
                  ReferencedProjects
-                 |> Seq.map (fun (FSharpReference(_name, p)) -> p.ProjectSnapshot.SignatureVersion)
+                 |> Seq.map (function
+                     | FSharpReference(_name, p) -> p.ProjectSnapshot.SignatureVersion
+                     | PEReference(getStamp, _) -> Md5Hasher.empty |> Md5Hasher.addDateTime (getStamp ())
+                     | ILModuleReference(_name, getStamp, _) -> Md5Hasher.empty |> Md5Hasher.addDateTime (getStamp ()))
              ))
 
     let fullHashString = lazy (fullHash.Value |> Md5Hasher.toString)
@@ -429,20 +432,42 @@ and internal ProjectCore
     member _.CacheKey = cacheKey.Value
 
 and [<NoComparison; CustomEquality; Experimental("This FCS API is experimental and subject to change.")>] FSharpReferencedProjectSnapshot =
-    | FSharpReference of projectOutputFile: string * options: FSharpProjectSnapshot
-    //| PEReference of projectOutputFile: string * getStamp: (unit -> DateTime) * delayedReader: DelayedILModuleReader
-    //| ILModuleReference of
-    //    projectOutputFile: string *
-    //    getStamp: (unit -> DateTime) *
-    //    getReader: (unit -> ILModuleReader)
+    /// <summary>
+    /// A reference to an F# project. The physical data for it is stored/cached inside of the compiler service.
+    /// </summary>
+    /// <param name="projectOutputFile">The fully qualified path to the output of the referenced project. This should be the same value as the <c>-r</c> reference in the project options for this referenced project.</param>
+    /// <param name="snapshot">Snapshot of the referenced F# project</param>
+    | FSharpReference of projectOutputFile: string * snapshot: FSharpProjectSnapshot
+    /// <summary>
+    /// A reference to any portable executable, including F#. The stream is owned by this reference.
+    /// The stream will be automatically disposed when there are no references to FSharpReferencedProject and is GC collected.
+    /// Once the stream is evaluated, the function that constructs the stream will no longer be referenced by anything.
+    /// If the stream evaluation throws an exception, it will be automatically handled.
+    /// </summary>
+    /// <param name="getStamp">A function that calculates a last-modified timestamp for this reference. This will be used to determine if the reference is up-to-date.</param>
+    /// <param name="delayedReader">A function that opens a Portable Executable data stream for reading.</param>
+    | PEReference of getStamp: (unit -> DateTime) * delayedReader: DelayedILModuleReader
+
+    /// <summary>
+    /// A reference to an ILModuleReader.
+    /// </summary>
+    /// <param name="projectOutputFile">The fully qualified path to the output of the referenced project. This should be the same value as the <c>-r</c> reference in the project options for this referenced project.</param>
+    /// <param name="getStamp">A function that calculates a last-modified timestamp for this reference. This will be used to determine if the reference is up-to-date.</param>
+    /// <param name="getReader">A function that creates an ILModuleReader for reading module data.</param>
+    | ILModuleReference of
+        projectOutputFile: string *
+        getStamp: (unit -> DateTime) *
+        getReader: (unit -> FSharp.Compiler.AbstractIL.ILBinaryReader.ILModuleReader)
 
     /// <summary>
     /// The fully qualified path to the output of the referenced project. This should be the same value as the <c>-r</c>
     /// reference in the project options for this referenced project.
     /// </summary>
     member this.OutputFile =
         match this with
-        | FSharpReference(projectOutputFile, _) -> projectOutputFile
+        | FSharpReference(projectOutputFile = projectOutputFile)
+        | ILModuleReference(projectOutputFile = projectOutputFile) -> projectOutputFile
+        | PEReference(delayedReader = reader) -> reader.OutputFile
 
     /// <summary>
     /// Creates a reference for an F# project. The physical data for it is stored/cached inside of the compiler service.
@@ -458,6 +483,11 @@ and [<NoComparison; CustomEquality; Experimental("This FCS API is experimental a
             match this, o with
             | FSharpReference(projectOutputFile1, options1), FSharpReference(projectOutputFile2, options2) ->
                 projectOutputFile1 = projectOutputFile2 && options1 = options2
+            | PEReference(getStamp1, reader1), PEReference(getStamp2, reader2) ->
+                reader1.OutputFile = reader2.OutputFile && (getStamp1 ()) = (getStamp2 ())
+            | ILModuleReference(projectOutputFile1, getStamp1, _), ILModuleReference(projectOutputFile2, getStamp2, _) ->
+                projectOutputFile1 = projectOutputFile2 && (getStamp1 ()) = (getStamp2 ())
+            | _ -> false
 
         | _ -> false
 
@@ -524,17 +554,19 @@ and [<Experimental("This FCS API is experimental and subject to change.")>] FSha
 
                 let! referencedProjects =
                     options.ReferencedProjects
-                    |> Seq.choose (function
+                    |> Seq.map (function
                         | FSharpReferencedProject.FSharpReference(outputName, options) ->
-                            Some(
-                                async {
-                                    let! snapshot = FSharpProjectSnapshot.FromOptions(options, getFileSnapshot, snapshotAccumulator)
-
-                                    return FSharpReferencedProjectSnapshot.FSharpReference(outputName, snapshot)
-                                }
-                            )
-                        // TODO: other types
-                        | _ -> None)
+                            async {
+                                let! snapshot = FSharpProjectSnapshot.FromOptions(options, getFileSnapshot, snapshotAccumulator)
+
+                                return FSharpReferencedProjectSnapshot.FSharpReference(outputName, snapshot)
+                            }
+                        | FSharpReferencedProject.PEReference(getStamp, reader) ->
+                            async.Return <| FSharpReferencedProjectSnapshot.PEReference(getStamp, reader)
+                        | FSharpReferencedProject.ILModuleReference(outputName, getStamp, getReader) ->
+                            async.Return
+                            <| FSharpReferencedProjectSnapshot.ILModuleReference(outputName, getStamp, getReader))
+
                     |> Async.Sequential
 
                 let referencesOnDisk, otherOptions =
@@ -601,7 +633,9 @@ let rec internal snapshotToOptions (projectSnapshot: ProjectSnapshotBase<_>) =
         ReferencedProjects =
             projectSnapshot.ReferencedProjects
             |> Seq.map (function
-                | FSharpReference(name, opts) -> FSharpReferencedProject.FSharpReference(name, opts.ProjectSnapshot |> snapshotToOptions))
+                | FSharpReference(name, opts) -> FSharpReferencedProject.FSharpReference(name, opts.ProjectSnapshot |> snapshotToOptions)
+                | PEReference(getStamp, reader) -> FSharpReferencedProject.PEReference(getStamp, reader)
+                | ILModuleReference(name, getStamp, getReader) -> FSharpReferencedProject.ILModuleReference(name, getStamp, getReader))
             |> Seq.toArray
         IsIncompleteTypeCheckEnvironment = projectSnapshot.IsIncompleteTypeCheckEnvironment
         UseScriptResolutionRules = projectSnapshot.UseScriptResolutionRules

src/Compiler/Service/TransparentCompiler.fs

@@ -539,6 +539,41 @@ type internal TransparentCompiler
 
                             member x.FileName = nm
                         }
+                | FSharpReferencedProjectSnapshot.PEReference(getStamp, delayedReader) ->
+                    { new IProjectReference with
+                        member x.EvaluateRawContents() =
+                            node {
+                                let! ilReaderOpt = delayedReader.TryGetILModuleReader() |> NodeCode.FromCancellable
+
+                                match ilReaderOpt with
+                                | Some ilReader ->
+                                    let ilModuleDef, ilAsmRefs = ilReader.ILModuleDef, ilReader.ILAssemblyRefs
+                                    let data = RawFSharpAssemblyData(ilModuleDef, ilAsmRefs) :> IRawFSharpAssemblyData
+                                    return ProjectAssemblyDataResult.Available data
+                                | _ ->
+                                    // Note 'false' - if a PEReference doesn't find an ILModuleReader then we don't
+                                    // continue to try to use an on-disk DLL
+                                    return ProjectAssemblyDataResult.Unavailable false
+                            }
+
+                        member x.TryGetLogicalTimeStamp _ = getStamp () |> Some
+                        member x.FileName = delayedReader.OutputFile
+                    }
+
+                | FSharpReferencedProjectSnapshot.ILModuleReference(nm, getStamp, getReader) ->
+                    { new IProjectReference with
+                        member x.EvaluateRawContents() =
+                            cancellable {
+                                let ilReader = getReader ()
+                                let ilModuleDef, ilAsmRefs = ilReader.ILModuleDef, ilReader.ILAssemblyRefs
+                                let data = RawFSharpAssemblyData(ilModuleDef, ilAsmRefs) :> IRawFSharpAssemblyData
+                                return ProjectAssemblyDataResult.Available data
+                            }
+                            |> NodeCode.FromCancellable
+
+                        member x.TryGetLogicalTimeStamp _ = getStamp () |> Some
+                        member x.FileName = nm
+                    }
         ]
 
     let ComputeTcConfigBuilder (projectSnapshot: ProjectSnapshotBase<_>) =
@@ -762,20 +797,23 @@ type internal TransparentCompiler
                 |> List.map (fun (m, fileName) -> m, FSharpFileSnapshot.CreateFromFileSystem(fileName))
 
             return
-                Some
-                    {
-                        Id = bootstrapId
-                        AssemblyName = assemblyName
-                        OutFile = outFile
-                        TcConfig = tcConfig
-                        TcImports = tcImports
-                        TcGlobals = tcGlobals
-                        InitialTcInfo = initialTcInfo
-                        LoadedSources = loadedSources
-                        LoadClosure = loadClosureOpt
-                        LastFileName = sourceFiles |> List.last
-                    //ImportsInvalidatedByTypeProvider = importsInvalidatedByTypeProvider
-                    }
+                match sourceFiles with
+                | [] -> None
+                | _ ->
+                    Some
+                        {
+                            Id = bootstrapId
+                            AssemblyName = assemblyName
+                            OutFile = outFile
+                            TcConfig = tcConfig
+                            TcImports = tcImports
+                            TcGlobals = tcGlobals
+                            InitialTcInfo = initialTcInfo
+                            LoadedSources = loadedSources
+                            LoadClosure = loadClosureOpt
+                            LastFileName = sourceFiles |> List.tryLast |> Option.defaultValue ""
+                        //ImportsInvalidatedByTypeProvider = importsInvalidatedByTypeProvider
+                        }
         }
 
     let ComputeBootstrapInfo (projectSnapshot: ProjectSnapshot) =
@@ -1112,7 +1150,7 @@ type internal TransparentCompiler
                 ApplyMetaCommandsFromInputToTcConfig(tcConfig, input, Path.GetDirectoryName fileName, tcImports.DependencyProvider)
                 |> ignore
 
-                let sink = TcResultsSinkImpl(tcGlobals)
+                let sink = TcResultsSinkImpl(tcGlobals, file.SourceText)
 
                 let hadParseErrors = not (Array.isEmpty file.ParseErrors)
 
@@ -1141,16 +1179,13 @@ type internal TransparentCompiler
 
                     //fileChecked.Trigger fileName
 
-                    let newErrors =
-                        Array.append file.ParseErrors (errHandler.CollectedPhasedDiagnostics)
-
                     fileChecked.Trigger(fileName, Unchecked.defaultof<_>)
 
                     return
                         {
                             finisher = finisher
                             moduleNamesDict = moduleNamesDict
-                            tcDiagnosticsRev = [ newErrors ]
+                            tcDiagnosticsRev = [ errHandler.CollectedPhasedDiagnostics ]
                             tcDependencyFiles = [ fileName ]
                             sink = sink
                         }
@@ -1353,7 +1388,7 @@ type internal TransparentCompiler
                     let! snapshotWithSources = LoadSources bootstrapInfo priorSnapshot
                     let file = snapshotWithSources.SourceFiles |> List.last
 
-                    let! parseResults = getParseResult projectSnapshot creationDiags file bootstrapInfo.TcConfig
+                    let! parseResults = getParseResult projectSnapshot Seq.empty file bootstrapInfo.TcConfig
 
                     let! result, tcInfo = ComputeTcLastFile bootstrapInfo snapshotWithSources
 
@@ -1405,8 +1440,7 @@ type internal TransparentCompiler
                             Some symbolEnv
                         )
 
-                    let tcDiagnostics =
-                        [| yield! creationDiags; yield! extraDiagnostics; yield! tcDiagnostics |]
+                    let tcDiagnostics = [| yield! extraDiagnostics; yield! tcDiagnostics |]
 
                     let loadClosure = None // TODO: script support
 
@@ -1649,7 +1683,8 @@ type internal TransparentCompiler
                         | ProjectAssemblyDataResult.Available data -> Some data
                         | _ -> None
 
-                    let symbolUses = tcInfo.sink |> Seq.map (fun sink -> sink.GetSymbolUses())
+                    let symbolUses =
+                        tcInfo.sink |> Seq.rev |> Seq.map (fun sink -> sink.GetSymbolUses())
 
                     let details =
                         (bootstrapInfo.TcGlobals,

tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs

@@ -479,12 +479,12 @@ type SignatureFiles = Yes = 1 | No = 2 | Some = 3
 let fuzzingTest seed (project: SyntheticProject) = task {
     let rng = System.Random seed
 
-    let checkingThreads = 3
-    let maxModificationDelayMs = 10
+    let checkingThreads = 10
+    let maxModificationDelayMs = 50
     let maxCheckingDelayMs = 20
     //let runTimeMs = 30000
     let signatureFileModificationProbability = 0.25
-    let modificationLoopIterations = 10
+    let modificationLoopIterations = 50
     let checkingLoopIterations = 5
 
     let minCheckingTimeoutMs = 0
@@ -622,7 +622,7 @@ let fuzzingTest seed (project: SyntheticProject) = task {
             }
 
         try
-            let! _x = threads |> Seq.skip 1 |> Task.WhenAll
+            let! _x = threads |> Task.WhenAll
             ()
         with
             | e ->

tests/FSharp.Compiler.ComponentTests/TypeChecks/Graph/TrieMappingTests.fs

@@ -540,6 +540,27 @@ let ``Tries are built up incrementally`` () =
                     ParsedInput = parseSourceCode ("D.fs", "module D")
                 }
             |]
-            
+
     for idx, t in trie do
         Assert.AreEqual(idx + 1, t.Children.Count)
+
+
+module InvalidSyntax =
+
+    [<Test>]
+    let ``Unnamed module`` () =
+        let trie =
+            getLastTrie
+                [| { Idx = 0
+                     FileName = "A.fs"
+                     ParsedInput =
+                       parseSourceCode (
+                           "A.fs",
+                           """
+                        module
+
+                        ()
+                        """
+                       ) } |]
+
+        Assert.True trie.Children.IsEmpty