Remove trailing semicolon analyzer & code fix and Refactor lexer logic in helpers into a MEF class

VisualFSharp.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 15
-VisualStudioVersion = 15.0.26009.0
+VisualStudioVersion = 15.0.26014.0
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Compiler", "src\fsharp\FSharp.Compiler\FSharp.Compiler.fsproj", "{2E4D67B4-522D-4CF7-97E4-BA940F0B18F3}"
 EndProject

src/fsharp/vs/ServiceUntypedParse.fsi

@@ -92,6 +92,8 @@ module internal UntypedParseImpl =
     val TryFindExpressionIslandInPosition : pos * ParsedInput option -> string option
     val TryGetCompletionContext : pos * FSharpParseFileResults option -> CompletionContext option
     val GetEntityKind: pos * ParsedInput -> EntityKind option
+    val GetLongIdents: ParsedInput option -> IDictionary<pos, LongIdent>
+    val GetLongIdentAt: ParsedInput * pos -> LongIdent option
 
 // implementation details used by other code in the compiler    
 module internal SourceFileImpl =

vsintegration/src/FSharp.Editor/Classification/ColorizationService.fs

@@ -23,7 +23,8 @@ type internal FSharpColorizationService
     [<ImportingConstructor>]
     (
         checkerProvider: FSharpCheckerProvider,
-        projectInfoManager: ProjectInfoManager
+        projectInfoManager: ProjectInfoManager,
+        lexer: Lexer
     ) =
     interface IEditorClassificationService with
         // Do not perform classification if we don't have project options (#defines matter)
@@ -33,7 +34,7 @@ type internal FSharpColorizationService
             async {
                 let defines = projectInfoManager.GetCompilationDefinesForEditingDocument(document)  
                 let! sourceText = document.GetTextAsync(cancellationToken)
-                result.AddRange(CommonHelpers.getColorizationData(document.Id, sourceText, textSpan, Some(document.FilePath), defines, cancellationToken))
+                result.AddRange(lexer.GetColorizationData(document.Id, sourceText, textSpan, Some(document.FilePath), defines, cancellationToken))
             } |> CommonRoslynHelpers.StartAsyncUnitAsTask cancellationToken
 
         member this.AddSemanticClassificationsAsync(document: Document, textSpan: TextSpan, result: List<ClassifiedSpan>, cancellationToken: CancellationToken) =
@@ -44,16 +45,13 @@ type internal FSharpColorizationService
                     let! textVersion = document.GetTextVersionAsync(cancellationToken)
                     let! _parseResults, checkResultsAnswer = checkerProvider.Checker.ParseAndCheckFileInProject(document.FilePath, textVersion.GetHashCode(), sourceText.ToString(), options)
 
-                    let extraColorizationData = 
-                        match checkResultsAnswer with
-                        | FSharpCheckFileAnswer.Aborted -> [| |]
-                        | FSharpCheckFileAnswer.Succeeded(results) -> 
-                            [| for (range, tokenColorKind)  in results.GetExtraColorizationsAlternate() do
-                                  let span = CommonHelpers.fixupSpan(sourceText, CommonRoslynHelpers.FSharpRangeToTextSpan(sourceText, range))
-                                  if textSpan.Contains(span.Start) || textSpan.Contains(span.End - 1) || span.Contains(textSpan) then
-                                      yield ClassifiedSpan(span, CommonHelpers.compilerTokenToRoslynToken(tokenColorKind)) |]
-
-                    result.AddRange(extraColorizationData)
+                    match checkResultsAnswer with
+                    | FSharpCheckFileAnswer.Aborted -> ()
+                    | FSharpCheckFileAnswer.Succeeded(results) -> 
+                        for (range, tokenColorKind)  in results.GetExtraColorizationsAlternate() do
+                           let span = CommonHelpers.fixupSpan(sourceText, CommonRoslynHelpers.FSharpRangeToTextSpan(sourceText, range))
+                           if textSpan.Contains(span.Start) || textSpan.Contains(span.End - 1) || span.Contains(textSpan) then
+                               result.Add(ClassifiedSpan(span, CommonHelpers.compilerTokenToRoslynToken(tokenColorKind)))
                 | None -> ()
             } |> CommonRoslynHelpers.StartAsyncUnitAsTask cancellationToken
 

vsintegration/src/FSharp.Editor/CodeFix/AddOpenCodeFixProvider.fs → vsintegration/src/FSharp.Editor/CodeFixes/AddOpen.fs

@@ -178,34 +178,37 @@ type internal FSharpAddOpenCodeFixProvider
                 | None, _
                 | _, FSharpCheckFileAnswer.Aborted -> ()
                 | Some parsedInput, FSharpCheckFileAnswer.Succeeded checkFileResults ->
-                    let textLinePos = sourceText.Lines.GetLinePosition context.Span.Start
-                    let defines = CompilerEnvironment.GetCompilationDefinesForEditing(context.Document.FilePath, options.OtherOptions |> Seq.toList)
-                    let symbol = CommonHelpers.getSymbolAtPosition(context.Document.Id, sourceText, context.Span.Start, context.Document.FilePath, defines, SymbolLookupKind.Fuzzy)
-                    match symbol with
-                    | Some symbol ->
-                        let pos = Pos.fromZ textLinePos.Line textLinePos.Character
-                        let isAttribute = UntypedParseImpl.GetEntityKind(pos, parsedInput) = Some EntityKind.Attribute
-                        let entities =
-                            assemblyContentProvider.GetAllEntitiesInProjectAndReferencedAssemblies checkFileResults
-                            |> List.map (fun e -> 
-                                 [ yield e.TopRequireQualifiedAccessParent, e.AutoOpenParent, e.Namespace, e.CleanedIdents
-                                   if isAttribute then
-                                       let lastIdent = e.CleanedIdents.[e.CleanedIdents.Length - 1]
-                                       if lastIdent.EndsWith "Attribute" && e.Kind LookupType.Precise = EntityKind.Attribute then
-                                           yield 
-                                               e.TopRequireQualifiedAccessParent, 
-                                               e.AutoOpenParent,
-                                               e.Namespace,
-                                               e.CleanedIdents 
-                                               |> Array.replace (e.CleanedIdents.Length - 1) (lastIdent.Substring(0, lastIdent.Length - 9)) ])
-                            |> List.concat
-
-                        let idents = ParsedInput.getLongIdentAt parsedInput (Range.mkPos pos.Line symbol.RightColumn)
-                        match idents with
-                        | Some idents ->
-                            let createEntity = ParsedInput.tryFindInsertionContext pos.Line parsedInput idents
-                            return entities |> Seq.map createEntity |> Seq.concat |> Seq.toList |> getSuggestions context
-                        | None -> ()
+                    let unresolvedIdentRange = CommonRoslynHelpers.TextSpanToFSharpRange(context.Document.FilePath, sourceText, context.Span)
+                    let isAttribute = UntypedParseImpl.GetEntityKind(unresolvedIdentRange.Start, parsedInput) = Some EntityKind.Attribute
+                    let entities =
+                        assemblyContentProvider.GetAllEntitiesInProjectAndReferencedAssemblies checkFileResults
+                        |> List.collect (fun e -> 
+                             [ yield e.TopRequireQualifiedAccessParent, e.AutoOpenParent, e.Namespace, e.CleanedIdents
+                               if isAttribute then
+                                   let lastIdent = e.CleanedIdents.[e.CleanedIdents.Length - 1]
+                                   if lastIdent.EndsWith "Attribute" && e.Kind LookupType.Precise = EntityKind.Attribute then
+                                       yield 
+                                           e.TopRequireQualifiedAccessParent, 
+                                           e.AutoOpenParent,
+                                           e.Namespace,
+                                           e.CleanedIdents 
+                                           |> Array.replace (e.CleanedIdents.Length - 1) (lastIdent.Substring(0, lastIdent.Length - 9)) ])
+
+                    let longIdent = UntypedParseImpl.GetLongIdentAt(parsedInput, unresolvedIdentRange.End)
+
+                    let maybeUnresolvedIdents =
+                        longIdent 
+                        |> Option.map (fun longIdent ->
+                            longIdent
+                            |> List.map (fun ident ->
+                                { Ident = ident.idText
+                                  Resolved = not (ident.idRange = unresolvedIdentRange) })
+                            |> List.toArray)
+
+                    match maybeUnresolvedIdents with
+                    | Some maybeUnresolvedIdents ->
+                        let createEntity = ParsedInput.tryFindInsertionContext unresolvedIdentRange.StartLine parsedInput maybeUnresolvedIdents
+                        return entities |> Seq.map createEntity |> Seq.concat |> Seq.toList |> getSuggestions context
                     | None -> ()
             | None -> ()
         } |> CommonRoslynHelpers.StartAsyncUnitAsTask(context.CancellationToken)

vsintegration/src/FSharp.Editor/CodeFix/ImplementInterfaceCodeFixProvider.fs → vsintegration/src/FSharp.Editor/CodeFixes/ImplementInterfaceCodeFixProvider.fs

@@ -31,7 +31,8 @@ type internal FSharpImplementInterfaceCodeFixProvider
     [<ImportingConstructor>]
     (
         checkerProvider: FSharpCheckerProvider, 
-        projectInfoManager: ProjectInfoManager
+        projectInfoManager: ProjectInfoManager,
+        lexer: Lexer
     ) =
     inherit CodeFixProvider()
     let fixableDiagnosticIds = ["FS0366"]
@@ -156,7 +157,7 @@ type internal FSharpImplementInterfaceCodeFixProvider
                     let defines = CompilerEnvironment.GetCompilationDefinesForEditing(context.Document.FilePath, options.OtherOptions |> Seq.toList)
                     // Notice that context.Span doesn't return reliable ranges to find tokens at exact positions.
                     // That's why we tokenize the line and try to find the last successive identifier token
-                    let tokens = CommonHelpers.tokenizeLine(context.Document.Id, sourceText, context.Span.Start, context.Document.FilePath, defines)
+                    let tokens = lexer.TokenizeLine(context.Document.Id, sourceText, context.Span.Start, context.Document.FilePath, defines, context.CancellationToken)
                     let startLeftColumn = context.Span.Start - textLine.Start
                     let rec tryFindIdentifierToken acc tokens =
                        match tokens with
@@ -180,7 +181,7 @@ type internal FSharpImplementInterfaceCodeFixProvider
                             | _ -> 
                                 Some context.Span.End
                         let! interfaceState = queryInterfaceState appendBracketAt interfacePos tokens parsedInput                        
-                        let symbol = CommonHelpers.getSymbolAtPosition(context.Document.Id, sourceText, fixupPosition, context.Document.FilePath, defines, SymbolLookupKind.Fuzzy)
+                        let symbol = lexer.GetSymbolAtPosition(context.Document.Id, sourceText, fixupPosition, context.Document.FilePath, defines, SymbolLookupKind.Fuzzy)
                         match interfaceState, symbol with
                         | Some state, Some symbol ->                                
                             let fcsTextLineNumber = textLine.LineNumber + 1

vsintegration/src/FSharp.Editor/CodeFix/ProposeUppercaseLabel.fs → vsintegration/src/FSharp.Editor/CodeFixes/ProposeUppercaseLabel.fs

@@ -13,7 +13,8 @@ type internal FSharpProposeUpperCaseLabelCodeFixProvider
     [<ImportingConstructor>]
     (
         checkerProvider: FSharpCheckerProvider, 
-        projectInfoManager: ProjectInfoManager
+        projectInfoManager: ProjectInfoManager,
+        lexer: Lexer
     ) =
     inherit CodeFixProvider()
     let fixableDiagnosticIds = ["FS0053"]
@@ -23,7 +24,8 @@ type internal FSharpProposeUpperCaseLabelCodeFixProvider
     override __.RegisterCodeFixesAsync context : Task =
         asyncMaybe {
             let textChanger (originalText: string) = originalText.[0].ToString().ToUpper() + originalText.Substring(1)
-            let! solutionChanger, originalText = SymbolHelpers.changeAllSymbolReferences(context.Document, context.Span, textChanger, projectInfoManager, checkerProvider.Checker)
+            let! solutionChanger, originalText = 
+                SymbolHelpers.changeAllSymbolReferences(context.Document, context.Span, textChanger, projectInfoManager, checkerProvider.Checker, lexer)
             let title = FSComp.SR.replaceWithSuggestion (textChanger originalText)
             context.RegisterCodeFix(
                 CodeAction.Create(title, solutionChanger, title),

vsintegration/src/FSharp.Editor/CodeFixes/RemoveTrailingSemicolon.fs

@@ -0,0 +1,38 @@
+// Copyright (c) Microsoft Corporation.  All Rights Reserved.  Licensed under the Apache License, Version 2.0.  See License.txt in the project root for license information.
+
+namespace rec Microsoft.VisualStudio.FSharp.Editor
+
+open System.Composition
+open System.Collections.Immutable
+open System.Threading
+open System.Threading.Tasks
+
+open Microsoft.CodeAnalysis
+open Microsoft.CodeAnalysis.Text
+open Microsoft.CodeAnalysis.CodeFixes
+open Microsoft.CodeAnalysis.CodeActions
+open Microsoft.CodeAnalysis.Editor
+open Microsoft.CodeAnalysis.Host.Mef
+
+[<ExportCodeFixProvider(FSharpCommonConstants.FSharpLanguageName, Name = "RemoveTrailingSemicolon"); Shared>]
+type internal RemoveTrailingSemicolonCodeFixProvider() =
+    inherit CodeFixProvider()
+    let fixableDiagnosticIds = [TrailingSemicolonDiagnosticAnalyzer.DiagnosticId]
+
+    let createCodeFix (title: string, context: CodeFixContext, textChange: TextChange) =
+        CodeAction.Create(
+            title,
+            (fun (cancellationToken: CancellationToken) ->
+                async {
+                    let! sourceText = context.Document.GetTextAsync() |> Async.AwaitTask
+                    return context.Document.WithText(sourceText.WithChanges(textChange))
+                } |> CommonRoslynHelpers.StartAsyncAsTask(cancellationToken)),
+            title)
+
+    override __.FixableDiagnosticIds = fixableDiagnosticIds.ToImmutableArray()
+
+    override __.RegisterCodeFixesAsync context : Task =
+       async {
+           let diagnostics = (context.Diagnostics |> Seq.filter (fun x -> fixableDiagnosticIds |> List.contains x.Id)).ToImmutableArray()
+           context.RegisterCodeFix(createCodeFix(SR.RemoveTrailingSemicolon.Value, context, TextChange(context.Span, "")), diagnostics)
+        } |> CommonRoslynHelpers.StartAsyncUnitAsTask(context.CancellationToken)

vsintegration/src/FSharp.Editor/Commands/HelpContextService.fs

@@ -21,7 +21,8 @@ type internal FSharpHelpContextService
     [<ImportingConstructor>]
     (
         checkerProvider: FSharpCheckerProvider,
-        projectInfoManager: ProjectInfoManager
+        projectInfoManager: ProjectInfoManager,
+        lexer: Lexer
     ) =
 
     static member GetHelpTerm(checker: FSharpChecker, sourceText : SourceText, fileName, options, span: TextSpan, tokens: List<ClassifiedSpan>, textVersion) = async {
@@ -112,7 +113,7 @@ type internal FSharpHelpContextService
                     let! textVersion = document.GetTextVersionAsync(cancellationToken)
                     let defines = projectInfoManager.GetCompilationDefinesForEditingDocument(document)  
                     let textLine = sourceText.Lines.GetLineFromPosition(textSpan.Start)
-                    let tokens = CommonHelpers.getColorizationData(document.Id, sourceText, textLine.Span, Some document.Name, defines, cancellationToken)
+                    let tokens = lexer.GetColorizationData(document.Id, sourceText, textLine.Span, Some document.Name, defines, cancellationToken)
                     let! keyword = FSharpHelpContextService.GetHelpTerm(checkerProvider.Checker, sourceText, document.FilePath, options, textSpan, tokens, textVersion.GetHashCode())
                     return defaultArg keyword String.Empty
                 | None -> return String.Empty