Update Roslyn to base off beta6 version of Roslyn + Type Colorization

.nuget/NuGet.Config

@@ -8,7 +8,7 @@
     <!--To inherit the global NuGet package sources remove the <clear/> line below -->
     <clear />
     <add key="myget.org fsharp-daily" value="https://www.myget.org/F/fsharp-daily/api/v3/index.json" />  
-    <add key="myget.org roslyn-dev15-preview4-df-nightly" value="https://dotnet.myget.org/F/roslyn-dev15-preview4-df-nightly/api/v3/index.json" />
+    <add key="myget.org roslyn-master-nightly" value="https://dotnet.myget.org/F/roslyn-master-nightly/api/v3/index.json" />
     <add key="dotnet-core" value="https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" />
     <add key="myget.org dotnet-buildtools" value="https://dotnet.myget.org/F/dotnet-buildtools/api/v3/index.json" />  
     <add key="myget.org roslyn-tools" value="https://dotnet.myget.org/F/roslyn-tools/api/v3/index.json" />  

packages.config

@@ -18,12 +18,12 @@
   <package id="WiX.Toolset.2015" version="3.10.0.1503" />
   <package id="Microsoft.VisualFSharp.Core.Redist" version="1.0.0" />
   <package id="Microsoft.VisualFSharp.Type.Providers.Redist" version="1.0.0" />
-  <package id="Microsoft.CodeAnalysis.Common" version="2.0.0-beta4-60808-03" targetFramework="net46" />
-  <package id="Microsoft.CodeAnalysis.EditorFeatures" version="2.0.0-beta4-60808-03" targetFramework="net46" />
-  <package id="Microsoft.CodeAnalysis.EditorFeatures.Text" version="2.0.0-beta4-60808-03" targetFramework="net46" />
-  <package id="Microsoft.CodeAnalysis.Features" version="2.0.0-beta4-60808-03" targetFramework="net46" />
-  <package id="Microsoft.CodeAnalysis.Workspaces.Common" version="2.0.0-beta4-60808-03" targetFramework="net46" />
-  <package id="Microsoft.VisualStudio.LanguageServices" version="2.0.0-beta4-60808-03" targetFramework="net46" />
+  <package id="Microsoft.CodeAnalysis.Common" version="2.0.0-beta6-61005-05" targetFramework="net46" />
+  <package id="Microsoft.CodeAnalysis.EditorFeatures" version="2.0.0-beta6-61005-05" targetFramework="net46" />
+  <package id="Microsoft.CodeAnalysis.EditorFeatures.Text" version="2.0.0-beta6-61005-05" targetFramework="net46" />
+  <package id="Microsoft.CodeAnalysis.Features" version="2.0.0-beta6-61005-05" targetFramework="net46" />
+  <package id="Microsoft.CodeAnalysis.Workspaces.Common" version="2.0.0-beta6-61005-05" targetFramework="net46" />
+  <package id="Microsoft.VisualStudio.LanguageServices" version="2.0.0-beta6-61005-05" targetFramework="net46" />
   <package id="Microsoft.Composition" version="1.0.27" targetFramework="net46" />
   <package id="Microsoft.VisualStudio.Threading" version="14.1.131" targetFramework="net46" />
   <package id="Microsoft.VisualStudio.Shell.14.0" version="14.3.25407" targetFramework="net46" />

src/FSharpSource.Settings.targets

@@ -90,7 +90,7 @@
     <FsiToolExe>fsi.exe</FsiToolExe>
     <FsLexToolExe>fslex.exe</FsLexToolExe>
     <FsYaccToolExe>fsyacc.exe</FsYaccToolExe>
-    <RoslynVersion>2.0.0-beta4-60808-03</RoslynVersion>
+    <RoslynVersion>2.0.0-beta6-61005-05</RoslynVersion>
     <RoslynVSBinariesVersion>14.0</RoslynVSBinariesVersion>
     <RoslynVSPackagesVersion>14.3.25407</RoslynVSPackagesVersion>
   </PropertyGroup>

vsintegration/src/FSharp.Editor/ColorizationService.fs

@@ -37,26 +37,26 @@ type private SourceTextData(lines: int) =
 type internal FSharpColorizationService() =
 
     static let DataCache = ConditionalWeakTable<SourceText, SourceTextData>()
+
+    static let compilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) : string = 
+        match colorKind with
+        | FSharpTokenColorKind.Comment -> ClassificationTypeNames.Comment
+        | FSharpTokenColorKind.Identifier -> ClassificationTypeNames.Identifier
+        | FSharpTokenColorKind.Keyword -> ClassificationTypeNames.Keyword
+        | FSharpTokenColorKind.String -> ClassificationTypeNames.StringLiteral
+        | FSharpTokenColorKind.Text -> ClassificationTypeNames.Text
+        | FSharpTokenColorKind.UpperIdentifier -> ClassificationTypeNames.Identifier
+        | FSharpTokenColorKind.Number -> ClassificationTypeNames.NumericLiteral
+        | FSharpTokenColorKind.InactiveCode -> ClassificationTypeNames.ExcludedCode 
+        | FSharpTokenColorKind.PreprocessorKeyword -> ClassificationTypeNames.PreprocessorKeyword 
+        | FSharpTokenColorKind.Operator -> ClassificationTypeNames.Operator
+        | FSharpTokenColorKind.TypeName  -> ClassificationTypeNames.ClassName
+        | FSharpTokenColorKind.Default | _ -> ClassificationTypeNames.Text
 
     static let scanSourceLine(sourceTokenizer: FSharpSourceTokenizer, textLine: TextLine, lineContents: string, lexState: FSharpTokenizerLexState) : SourceLineData =
 
         let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.Text
         let lineTokenizer = sourceTokenizer.CreateLineTokenizer(lineContents)
-
-        let compilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) : string = 
-            match colorKind with
-            | FSharpTokenColorKind.Comment -> ClassificationTypeNames.Comment
-            | FSharpTokenColorKind.Identifier -> ClassificationTypeNames.Identifier
-            | FSharpTokenColorKind.Keyword -> ClassificationTypeNames.Keyword
-            | FSharpTokenColorKind.String -> ClassificationTypeNames.StringLiteral
-            | FSharpTokenColorKind.Text -> ClassificationTypeNames.Text
-            | FSharpTokenColorKind.UpperIdentifier -> ClassificationTypeNames.Identifier
-            | FSharpTokenColorKind.Number -> ClassificationTypeNames.NumericLiteral
-            | FSharpTokenColorKind.InactiveCode -> ClassificationTypeNames.ExcludedCode 
-            | FSharpTokenColorKind.PreprocessorKeyword -> ClassificationTypeNames.PreprocessorKeyword 
-            | FSharpTokenColorKind.Operator -> ClassificationTypeNames.Operator
-            | FSharpTokenColorKind.TypeName  -> ClassificationTypeNames.ClassName
-            | FSharpTokenColorKind.Default | _ -> ClassificationTypeNames.Text
 
         let scanAndColorNextToken(lineTokenizer: FSharpLineTokenizer, lexState: Ref<FSharpTokenizerLexState>) : Option<FSharpTokenInfo> =
             let tokenInfoOption, nextLexState = lineTokenizer.ScanToken(lexState.Value)
@@ -144,17 +144,14 @@ type internal FSharpColorizationService() =
                     | None -> ()
                 , cancellationToken)
 
-        // FSROSLYNTODO: Due to issue 12732 on Roslyn side, semantic classification is tied to C#/VB only.
-        // Once that is exposed to F#, enable the below code path, and add tests accourdingly.
-        member this.AddSemanticClassificationsAsync(_, _, _, _) =
-            (*
+        member this.AddSemanticClassificationsAsync(document: Document, textSpan: TextSpan, result: List<ClassifiedSpan>, cancellationToken: CancellationToken) =
             let computation = async {
-                try
-                    let options = CommonRoslynHelpers.GetFSharpProjectOptionsForRoslynProject(document.Project)
+                match FSharpLanguageService.GetOptions(document.Project.Id) with
+                | Some(options) ->
                     let! sourceText = document.GetTextAsync(cancellationToken) |> Async.AwaitTask
                     let! parseResults = FSharpChecker.Instance.ParseFileInProject(document.Name, sourceText.ToString(), options)
                     let! textVersion = document.GetTextVersionAsync(cancellationToken) |> Async.AwaitTask
-                    let! checkResultsAnswer = FSharpChecker.Instance.CheckFileInProject(parseResults, document.Name, textVersion.GetHashCode(), textSpan.ToString(), options)
+                    let! checkResultsAnswer = FSharpChecker.Instance.CheckFileInProject(parseResults, document.FilePath, textVersion.GetHashCode(), textSpan.ToString(), options)
 
                     let extraColorizationData = match checkResultsAnswer with
                                                 | FSharpCheckFileAnswer.Aborted -> failwith "Compilation isn't complete yet"
@@ -163,13 +160,10 @@ type internal FSharpColorizationService() =
                                                 |> Seq.toList
 
                     result.AddRange(extraColorizationData)
-                with ex ->
-                    Assert.Exception(ex)
-                    raise(ex)
+                | None -> ()
             }
-            Task.Run(fun () -> Async.RunSynchronously(computation, cancellationToken = cancellationToken))
-            *)
-            Task.CompletedTask
-
+
+            Task.Run(CommonRoslynHelpers.GetTaskAction(computation), cancellationToken)
+
         // Do not perform classification if we don't have project options (#defines matter)
         member this.AdjustStaleClassification(_: SourceText, classifiedSpan: ClassifiedSpan) : ClassifiedSpan = classifiedSpan

vsintegration/src/FSharp.Editor/DocumentDiagnosticAnalyzer.fs

@@ -31,10 +31,11 @@ type internal FSharpDocumentDiagnosticAnalyzer() =
         let severity = if error.Severity = FSharpErrorSeverity.Error then DiagnosticSeverity.Error else DiagnosticSeverity.Warning
         let descriptor = new DiagnosticDescriptor(id, emptyString, description, error.Subcategory, severity, true, emptyString, String.Empty, null)
 
-        let linePositionSpan = LinePositionSpan(LinePosition(error.StartLineAlternate - 1, error.StartColumn), LinePosition(error.EndLineAlternate - 1, error.EndColumn))
+        // F# compiler report errors at start/end of file if parsing fails. It should be corrected to match Roslyn boundaries
+        let linePositionSpan = LinePositionSpan(
+                                LinePosition(Math.Max(0, error.StartLineAlternate - 1), error.StartColumn),
+                                LinePosition(Math.Max(0, error.EndLineAlternate - 1), error.EndColumn))
         let textSpan = sourceText.Lines.GetTextSpan(linePositionSpan)
-
-        // F# compiler report errors at end of file if parsing fails. It should be corrected to match Roslyn boundaries
         let correctedTextSpan = if textSpan.End < sourceText.Length then textSpan else TextSpan.FromBounds(sourceText.Length - 1, sourceText.Length)
 
         Diagnostic.Create(descriptor, Location.Create(filePath, correctedTextSpan , linePositionSpan))

vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj

@@ -32,7 +32,6 @@
     <Compile Include="AssemblyInfo.fs" />
     <Compile Include="CommonConstants.fs" />
     <Compile Include="CommonRoslynHelpers.fs" />
-    <Compile Include="ProjectSite.fs" />
     <Compile Include="LanguageService.fs" />
     <Compile Include="ColorizationService.fs">
       <Link>Classification\ColorizationService.fs</Link>
@@ -49,9 +48,6 @@
     <Compile Include="LanguageDebugInfoService.fs">
       <Link>Debugging\LanguageDebugInfoService.fs</Link>
     </Compile>
-    <Compile Include="DocumentDifferenceService.fs">
-      <Link>Diagnostics\DocumentDifferenceService.fs</Link>
-    </Compile>
     <Compile Include="DocumentDiagnosticAnalyzer.fs">
       <Link>Diagnostics\DocumentDiagnosticAnalyzer.fs</Link>
     </Compile>

vsintegration/src/FSharp.Editor/GoToDefinitionService.fs

@@ -16,7 +16,7 @@ open Microsoft.CodeAnalysis
 open Microsoft.CodeAnalysis.Classification
 open Microsoft.CodeAnalysis.Editor
 open Microsoft.CodeAnalysis.Editor.Host
-open Microsoft.CodeAnalysis.Editor.Navigation
+open Microsoft.CodeAnalysis.Navigation
 open Microsoft.CodeAnalysis.Editor.Shared.Utilities
 open Microsoft.CodeAnalysis.Host.Mef
 open Microsoft.CodeAnalysis.Text
@@ -29,12 +29,15 @@ open Microsoft.FSharp.Compiler.Range
 open Microsoft.FSharp.Compiler.SourceCodeServices
 
 type internal FSharpNavigableItem(document: Document, textSpan: TextSpan, displayString: string) =
+    member this.DisplayString = displayString
+
     interface INavigableItem with
         member this.Glyph = Glyph.BasicFile
         member this.DisplayFileLocation = true
-        member this.DisplayString = displayString
+        member this.IsImplicitlyDeclared = false
         member this.Document = document
         member this.SourceSpan = textSpan
+        member this.DisplayTaggedParts = Unchecked.defaultof<ImmutableArray<TaggedText>>
         member this.ChildItems = ImmutableArray<INavigableItem>.Empty
 
 [<Shared>]
@@ -124,7 +127,7 @@ type internal FSharpGoToDefinitionService [<ImportingConstructor>] ([<ImportMany
 
             if definitionTask.Status = TaskStatus.RanToCompletion then
                 if definitionTask.Result.Any() then
-                    let navigableItem = definitionTask.Result.First() // F# API provides only one INavigableItem
+                    let navigableItem = definitionTask.Result.First() :?> FSharpNavigableItem // F# API provides only one INavigableItem
                     for presenter in presenters do
                         presenter.DisplayResult(navigableItem.DisplayString, definitionTask.Result)
                     true

vsintegration/src/FSharp.Editor/IndentationService.fs

@@ -48,11 +48,14 @@ type internal FSharpIndentationService() =
 
     interface ISynchronousIndentationService with
         member this.GetDesiredIndentation(document: Document, lineNumber: int, cancellationToken: CancellationToken): Nullable<IndentationResult> =
-            let sourceTextTask= document.GetTextAsync(cancellationToken)
+            let sourceTextTask = document.GetTextAsync(cancellationToken)
             sourceTextTask.Wait(cancellationToken)
-
             let sourceText = CommonRoslynHelpers.GetCompletedTaskResult(sourceTextTask)
-            let tabSize = document.Options.GetOption(FormattingOptions.TabSize, FSharpCommonConstants.FSharpLanguageName)
+
+            let optionsTask = document.GetOptionsAsync(cancellationToken)
+            optionsTask.Wait(cancellationToken)
+            let options = CommonRoslynHelpers.GetCompletedTaskResult(optionsTask)
+            let tabSize = options.GetOption(FormattingOptions.TabSize, FSharpCommonConstants.FSharpLanguageName)
 
             match FSharpIndentationService.GetDesiredIndentation(sourceText, lineNumber, tabSize) with
             | None -> Nullable<IndentationResult>()

vsintegration/src/FSharp.Editor/LanguageService.fs

@@ -5,6 +5,8 @@ namespace Microsoft.VisualStudio.FSharp.Editor
 open System
 open System.Collections.Generic
 open System.Runtime.InteropServices
+open System.Linq
+open System.IO
 
 open Microsoft.FSharp.Compiler.SourceCodeServices
 
@@ -15,7 +17,9 @@ open Microsoft.VisualStudio.LanguageServices
 open Microsoft.VisualStudio.LanguageServices.Implementation.LanguageService
 open Microsoft.VisualStudio.LanguageServices.Implementation.ProjectSystem
 open Microsoft.VisualStudio.LanguageServices.Implementation.DebuggerIntelliSense
+open Microsoft.VisualStudio.LanguageServices.Implementation.TaskList
 open Microsoft.VisualStudio.LanguageServices.Implementation
+open Microsoft.VisualStudio.LanguageServices.ProjectSystem
 open Microsoft.VisualStudio.Shell
 open Microsoft.VisualStudio.Shell.Interop
 open Microsoft.VisualStudio.FSharp.LanguageService
@@ -27,7 +31,7 @@ type internal SVsSettingsPersistenceManager = class end
 
 [<Guid(FSharpCommonConstants.languageServiceGuidString)>]
 type internal FSharpLanguageService(package : FSharpPackage) = 
-    inherit AbstractLanguageService<FSharpPackage, FSharpLanguageService, FSharpProjectSite>(package)
+    inherit AbstractLanguageService<FSharpPackage, FSharpLanguageService>(package)
 
     static let optionsCache = Dictionary<ProjectId, FSharpProjectOptions>()
     static member GetOptions(projectId: ProjectId) =
@@ -36,6 +40,13 @@ type internal FSharpLanguageService(package : FSharpPackage) =
         else
             None
 
+    member this.SyncProject(project: AbstractProject, projectContext: IWorkspaceProjectContext, site: IProjectSite) =
+        let updatedFiles = site.SourceFilesOnDisk()
+        let workspaceFiles = project.GetCurrentDocuments() |> Seq.map(fun file -> file.FilePath)
+
+        for file in updatedFiles do if not(workspaceFiles.Contains(file)) then projectContext.AddSourceFile(file)
+        for file in workspaceFiles do if not(updatedFiles.Contains(file)) then projectContext.RemoveSourceFile(file)
+
     override this.ContentTypeName = FSharpCommonConstants.FSharpContentTypeName
     override this.LanguageName = FSharpCommonConstants.FSharpLanguageName
     override this.RoslynLanguageName = FSharpCommonConstants.FSharpLanguageName
@@ -52,7 +63,6 @@ type internal FSharpLanguageService(package : FSharpPackage) =
         // FSROSLYNTODO: Hide navigation bars for now. Enable after adding tests
         workspace.Options <- workspace.Options.WithChangedOption(NavigationBarOptions.ShowNavigationBar, FSharpCommonConstants.FSharpLanguageName, false)
 
-        // Ensure that we have a project in the workspace for this document.
         let (_, buffer) = view.GetBuffer()
         let filename = VsTextLines.GetFilename buffer
         let result = VsRunningDocumentTable.FindDocumentWithoutLocking(package.RunningDocumentTable,filename)
@@ -61,6 +71,7 @@ type internal FSharpLanguageService(package : FSharpPackage) =
             match hier with
             | :? IProvideProjectSite as siteProvider ->
                 let site = siteProvider.GetProjectSite()
+                let projectGuid = Guid(site.ProjectGuid)
                 let projectFileName = site.ProjectFileName()
                 let projectId = workspace.ProjectTracker.GetOrCreateProjectIdForPath(projectFileName, projectFileName)
 
@@ -69,14 +80,26 @@ type internal FSharpLanguageService(package : FSharpPackage) =
                     optionsCache.Add(projectId, options)
 
                 if obj.ReferenceEquals(workspace.ProjectTracker.GetProject(projectId), null) then
-                    let projectSite = new FSharpProjectSite(hier, this.SystemServiceProvider, workspace, projectFileName);
-                    projectSite.Initialize(hier, site)                    
+                    let projectContextFactory = this.Package.ComponentModel.GetService<IWorkspaceProjectContextFactory>();
+                    let errorReporter = ProjectExternalErrorReporter(projectId, "FS", this.SystemServiceProvider)
+                    let outputFlag = site.CompilerFlags() |> Seq.pick(fun flag ->
+                        if flag.StartsWith("-o:") then Some(flag.Substring(3))
+                        else if flag.StartsWith("--out:") then Some(flag.Substring(6))
+                        else None)
+                    let outputPath = if Path.IsPathRooted(outputFlag) then outputFlag else Path.Combine(Path.GetDirectoryName(projectFileName), outputFlag)
+
+                    let projectContext = projectContextFactory.CreateProjectContext(FSharpCommonConstants.FSharpLanguageName, projectFileName, projectFileName, projectGuid, hier, outputPath, errorReporter)
+                    let project = projectContext :?> AbstractProject
+
+                    this.SyncProject(project, projectContext, site)
+                    site.AdviseProjectSiteChanges(FSharpCommonConstants.FSharpLanguageServiceCallbackName, AdviseProjectSiteChanges(fun () -> this.SyncProject(project, projectContext, site)))
+                    site.AdviseProjectSiteClosed(FSharpCommonConstants.FSharpLanguageServiceCallbackName, AdviseProjectSiteChanges(fun () -> project.Disconnect()))
             | _ -> ()
         | _ -> ()
 
 and [<Guid(FSharpCommonConstants.packageGuidString)>]
     internal FSharpPackage() = 
-    inherit AbstractPackage<FSharpPackage, FSharpLanguageService, FSharpProjectSite>()
+    inherit AbstractPackage<FSharpPackage, FSharpLanguageService>()
 
     override this.RoslynLanguageName = FSharpCommonConstants.FSharpLanguageName