From 534626a2f347933c961da31a81ff6840fbf3672c Mon Sep 17 00:00:00 2001 From: Omar Tawfik Date: Tue, 23 Feb 2016 14:59:17 -0800 Subject: [PATCH 1/9] Removing EditorFactory for now, fixed ServiceProvider exception --- src/fsharp/vs/IncrementalBuild.fs | 10 +++++--- src/fsharp/vs/IncrementalBuild.fsi | 3 ++- src/fsharp/vs/ServiceLexing.fs | 8 ++++-- src/fsharp/vs/ServiceLexing.fsi | 3 +++ src/fsharp/vs/service.fsi | 3 +++ .../src/FSharp.Editor/FSharp.Editor.fsproj | 5 +++- .../FSharp.LanguageService.fsproj | 1 + .../FSharpCommonConstants.fs | 4 --- .../FSharpLanguageService.fs | 25 ++----------------- 9 files changed, 27 insertions(+), 35 deletions(-) diff --git a/src/fsharp/vs/IncrementalBuild.fs b/src/fsharp/vs/IncrementalBuild.fs index e4e7ce807da..229367e9084 100755 --- a/src/fsharp/vs/IncrementalBuild.fs +++ b/src/fsharp/vs/IncrementalBuild.fs @@ -927,7 +927,8 @@ type FSharpErrorSeverity = | Warning | Error -type FSharpErrorInfo(fileName, s:pos, e:pos, severity: FSharpErrorSeverity, message: string, subcategory: string, errorNum: int) = +type FSharpErrorInfo(id: string, fileName, s:pos, e:pos, severity: FSharpErrorSeverity, message: string, subcategory: string, errorNum: int) = + member __.Id = id member __.StartLine = Line.toZ s.Line member __.StartLineAlternate = s.Line member __.EndLine = Line.toZ e.Line @@ -939,17 +940,18 @@ type FSharpErrorInfo(fileName, s:pos, e:pos, severity: FSharpErrorSeverity, mess member __.Subcategory = subcategory member __.FileName = fileName member __.ErrorNumber = errorNum - member __.WithStart(newStart) = FSharpErrorInfo(fileName, newStart, e, severity, message, subcategory, errorNum) - member __.WithEnd(newEnd) = FSharpErrorInfo(fileName, s, newEnd, severity, message, subcategory, errorNum) + member __.WithStart(newStart) = FSharpErrorInfo(id, fileName, newStart, e, severity, message, subcategory, errorNum) + member __.WithEnd(newEnd) = FSharpErrorInfo(id, fileName, s, newEnd, severity, message, subcategory, errorNum) override __.ToString()= sprintf "%s (%d,%d)-(%d,%d) %s %s %s" fileName (int s.Line) (s.Column + 1) (int e.Line) (e.Column + 1) subcategory (if severity=FSharpErrorSeverity.Warning then "warning" else "error") message /// Decompose a warning or error into parts: position, severity, message, error number static member (*internal*) CreateFromException(exn,warn,trim:bool,fallbackRange:range) = + let id = "FS" + GetErrorNumber(exn).ToString() let m = match GetRangeOfError exn with Some m -> m | None -> fallbackRange let e = if trim then m.Start else m.End let msg = bufs (fun buf -> OutputPhasedError buf exn false) let errorNum = GetErrorNumber exn - FSharpErrorInfo(m.FileName, m.Start, e, (if warn then FSharpErrorSeverity.Warning else FSharpErrorSeverity.Error), msg, exn.Subcategory(), errorNum) + FSharpErrorInfo(id, m.FileName, m.Start, e, (if warn then FSharpErrorSeverity.Warning else FSharpErrorSeverity.Error), msg, exn.Subcategory(), errorNum) /// Decompose a warning or error into parts: position, severity, message, error number static member internal CreateFromExceptionAndAdjustEof(exn,warn,trim:bool,fallbackRange:range, (linesCount:int, lastLength:int)) = diff --git a/src/fsharp/vs/IncrementalBuild.fsi b/src/fsharp/vs/IncrementalBuild.fsi index f78637d67bb..52851dc8d9e 100755 --- a/src/fsharp/vs/IncrementalBuild.fsi +++ b/src/fsharp/vs/IncrementalBuild.fsi @@ -19,7 +19,8 @@ type internal FSharpErrorSeverity = | Error [] -type internal FSharpErrorInfo = +type internal FSharpErrorInfo = + member Id: string member FileName: string member StartLineAlternate:int member EndLineAlternate:int diff --git a/src/fsharp/vs/ServiceLexing.fs b/src/fsharp/vs/ServiceLexing.fs index 94c965792d0..ea6127f214d 100755 --- a/src/fsharp/vs/ServiceLexing.fs +++ b/src/fsharp/vs/ServiceLexing.fs @@ -485,7 +485,9 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, let skip = false // don't skip whitespace in the lexer let mutable singleLineTokenState = SingleLineTokenState.BeforeHash - let fsx = CompileOps.IsScript(filename) + let fsx = match filename with + | null -> false + | _ -> CompileOps.IsScript(filename) // ---------------------------------------------------------------------------------- // This implements post-processing of #directive tokens - not very elegant, but it works... @@ -552,7 +554,9 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, - do resetLexbufPos filename lexbuf + do match filename with + | null -> lexbuf.EndPos <- Internal.Utilities.Text.Lexing.Position.Empty + | _ -> resetLexbufPos filename lexbuf member x.ScanToken(lexintInitial) : Option * FSharpTokenizerLexState = use unwindBP = PushThreadBuildPhaseUntilUnwind (BuildPhase.Parse) diff --git a/src/fsharp/vs/ServiceLexing.fsi b/src/fsharp/vs/ServiceLexing.fsi index 786bcad0995..57cea90964e 100755 --- a/src/fsharp/vs/ServiceLexing.fsi +++ b/src/fsharp/vs/ServiceLexing.fsi @@ -15,6 +15,9 @@ open System.Collections.Generic open Microsoft.FSharp.Compiler open Microsoft.FSharp.Compiler.Range +type Position = int * int +type Range = Position * Position + /// Represents encoded information for the end-of-line continutation of lexing type FSharpTokenizerLexState = int64 diff --git a/src/fsharp/vs/service.fsi b/src/fsharp/vs/service.fsi index 2b606577175..1c3a12fba0e 100755 --- a/src/fsharp/vs/service.fsi +++ b/src/fsharp/vs/service.fsi @@ -519,6 +519,9 @@ type internal FSharpChecker = /// This function is called when the configuration is known to have changed for reasons not encoded in the ProjectOptions. /// For example, dependent references may have been deleted or created. member InvalidateConfiguration: options: FSharpProjectOptions -> unit + + /// Begin background parsing the given project. + member StartBackgroundCompile: options: FSharpProjectOptions -> unit /// Set the project to be checked in the background. Overrides any previous call to CheckProjectInBackground member CheckProjectInBackground: options: FSharpProjectOptions -> unit diff --git a/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj b/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj index 39544c4d460..7a24d3e4ec7 100644 --- a/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj +++ b/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj @@ -64,10 +64,13 @@ - + + + + $(FSharpSourcesRoot)\..\packages\Microsoft.Composition.$(MicrosoftCompositionVersion)\lib\portable-net45+win8+wp8+wpa81\System.Composition.AttributedModel.dll diff --git a/vsintegration/src/FSharp.LanguageService/FSharp.LanguageService.fsproj b/vsintegration/src/FSharp.LanguageService/FSharp.LanguageService.fsproj index e0b5e612e84..0709a5d4090 100644 --- a/vsintegration/src/FSharp.LanguageService/FSharp.LanguageService.fsproj +++ b/vsintegration/src/FSharp.LanguageService/FSharp.LanguageService.fsproj @@ -65,6 +65,7 @@ + diff --git a/vsintegration/src/FSharp.LanguageService/FSharpCommonConstants.fs b/vsintegration/src/FSharp.LanguageService/FSharpCommonConstants.fs index b1133e186eb..bdf6aac3bd0 100644 --- a/vsintegration/src/FSharp.LanguageService/FSharpCommonConstants.fs +++ b/vsintegration/src/FSharp.LanguageService/FSharpCommonConstants.fs @@ -12,10 +12,6 @@ module internal FSharpCommonConstants = [] let languageServiceGuidString = "BC6DD5A5-D4D6-4dab-A00D-A51242DBAF1B" [] - let editorFactoryGuidString = "4EB7CCB7-4336-4FFD-B12B-396E9FD079A9" - [] - let codePageEditorFactoryGuidString = "82A16493-EF43-47E0-B42D-D87BAAB5335D" - [] let svsSettingsPersistenceManagerGuidString = "9B164E40-C3A2-4363-9BC5-EB4039DEF653" [] let FSharpLanguageName = "F#" diff --git a/vsintegration/src/FSharp.LanguageService/FSharpLanguageService.fs b/vsintegration/src/FSharp.LanguageService/FSharpLanguageService.fs index ec6522776fc..ac444a6f333 100644 --- a/vsintegration/src/FSharp.LanguageService/FSharpLanguageService.fs +++ b/vsintegration/src/FSharp.LanguageService/FSharpLanguageService.fs @@ -39,12 +39,11 @@ type internal FSharpLanguageService(package : FSharpPackage) = override this.SetupNewTextView(view) = base.SetupNewTextView(view) let workspace = this.Package.ComponentModel.GetService(); - let sp = new ServiceProvider(this.SystemServiceProvider.GetService()) // Ensure that we have a project in the workspace for this document. let (_, buffer) = view.GetBuffer() let filename = VsTextLines.GetFilename buffer - let result = VsRunningDocumentTable.FindDocumentWithoutLocking(sp.RunningDocumentTable,filename) + let result = VsRunningDocumentTable.FindDocumentWithoutLocking(package.RunningDocumentTable,filename) match result with | Some (hier, _) -> match hier with @@ -58,17 +57,6 @@ type internal FSharpLanguageService(package : FSharpPackage) = | _ -> () | _ -> () -and [] - internal FSharpEditorFactory(package : FSharpPackage) = - inherit AbstractEditorFactory(package) - - override this.ContentTypeName = FSharpCommonConstants.FSharpContentTypeName - override this.GetFormattedTextChanges(_, _, _, _) = upcast Array.empty - -and [] - internal FSharpCodePageEditorFactory(editorFactory: FSharpEditorFactory) = - inherit AbstractCodePageEditorFactory(editorFactory) - and [] internal FSharpPackage() = inherit AbstractPackage() @@ -83,16 +71,7 @@ and [] override this.CreateLanguageService() = new FSharpLanguageService(this) - override this.CreateEditorFactories() = - // Disabling editor factories until fully implemented - - // let editorFactory = new FSharpEditorFactory(this) - // let codePageEditorFactory = new FSharpCodePageEditorFactory(editorFactory) - - [| - // editorFactory :> IVsEditorFactory; - // codePageEditorFactory :> IVsEditorFactory; - |] :> seq + override this.CreateEditorFactories() = Seq.empty override this.RegisterMiscellaneousFilesWorkspaceInformation(_) = () From 5720e91f5731fc44be0dcd987afdb6b192749c5c Mon Sep 17 00:00:00 2001 From: Omar Tawfik Date: Wed, 24 Feb 2016 19:41:46 -0800 Subject: [PATCH 2/9] Added colorization (without type colors) --- src/fsharp/vs/ServiceLexing.fs | 2 - src/fsharp/vs/ServiceLexing.fsi | 2 - src/fsharp/vs/service.fs | 2 - .../src/FSharp.Editor/FSharp.Editor.fsproj | 1 + .../FSharpColorizationService.fs | 116 ++++++++++++++++++ 5 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 vsintegration/src/FSharp.Editor/FSharpColorizationService.fs diff --git a/src/fsharp/vs/ServiceLexing.fs b/src/fsharp/vs/ServiceLexing.fs index ea6127f214d..bd438c44bea 100755 --- a/src/fsharp/vs/ServiceLexing.fs +++ b/src/fsharp/vs/ServiceLexing.fs @@ -88,9 +88,7 @@ type FSharpTokenColorKind = | PreprocessorKeyword = 8 | Number = 9 | Operator = 10 -#if COLORIZE_TYPES | TypeName = 11 -#endif /// Categorize an action the editor should take in response to a token, e.g. brace matching /// diff --git a/src/fsharp/vs/ServiceLexing.fsi b/src/fsharp/vs/ServiceLexing.fsi index 57cea90964e..620ef646f85 100755 --- a/src/fsharp/vs/ServiceLexing.fsi +++ b/src/fsharp/vs/ServiceLexing.fsi @@ -52,9 +52,7 @@ type internal FSharpTokenColorKind = | PreprocessorKeyword = 8 | Number = 9 | Operator = 10 -#if COLORIZE_TYPES | TypeName = 11 -#endif /// Gives an indication of what should happen when the token is typed in an IDE type internal FSharpTokenTriggerClass = diff --git a/src/fsharp/vs/service.fs b/src/fsharp/vs/service.fs index fab79549efd..9b4ffbf110b 100755 --- a/src/fsharp/vs/service.fs +++ b/src/fsharp/vs/service.fs @@ -1473,12 +1473,10 @@ type TypeCheckInfo // custom builders, custom operations get colored as keywords | CNR(_, (Item.CustomBuilder _ | Item.CustomOperation _), ItemOccurence.Use, _, _, _, m) -> yield (m, FSharpTokenColorKind.Keyword) -#if COLORIZE_TYPES // types get colored as types when they occur in syntactic types or custom attributes // typevariables get colored as types when they occur in syntactic types custom builders, custom operations get colored as keywords | CNR(_, (Item.TypeVar _ | Item.Types _ | Item.UnqualifiedType _) , (ItemOccurence.UseInType | ItemOccurence.UseInAttribute), _, _, _, m) -> yield (m, FSharpTokenColorKind.TypeName) -#endif | _ -> () |] member x.ScopeResolutions = sResolutions diff --git a/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj b/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj index 7a24d3e4ec7..8f1d66ddb23 100644 --- a/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj +++ b/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj @@ -30,6 +30,7 @@ + diff --git a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs new file mode 100644 index 00000000000..ea4f71fbec0 --- /dev/null +++ b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Microsoft.VisualStudio.FSharp.Editor + +open System +open System.Composition +open System.Collections.Concurrent +open System.Collections.Generic +open System.Threading +open System.Threading.Tasks +open System.Linq + +open Microsoft.CodeAnalysis +open Microsoft.CodeAnalysis.Classification +open Microsoft.CodeAnalysis.Editor +open Microsoft.CodeAnalysis.Editor.Implementation.Classification +open Microsoft.CodeAnalysis.Editor.Shared.Utilities +open Microsoft.CodeAnalysis.Host.Mef +open Microsoft.CodeAnalysis.Text + +open Microsoft.VisualStudio.FSharp.LanguageService +open Microsoft.VisualStudio.Text +open Microsoft.VisualStudio.Text.Tagging + +open Microsoft.FSharp.Compiler.SourceCodeServices + +[, FSharpCommonConstants.FSharpLanguageName)>] +type internal FSharpColorizationService() = + + // Create expensive resources upfront + let sourceTokenizer = FSharpSourceTokenizer([], String.Empty) + let lineEndingStates = new ConcurrentDictionary() + + interface IEditorClassificationService with + + member this.AddLexicalClassifications(text: SourceText, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = + this.ClassifySourceTextAsync(text, textSpan, result, cancellationToken).Wait(cancellationToken) + + member this.AddSyntacticClassificationsAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = + this.ClassifyDocumentAsync(document, textSpan, result, cancellationToken) + + member this.AddSemanticClassificationsAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = + this.ClassifyDocumentAsync(document, textSpan, result, cancellationToken) + + member this.AdjustStaleClassification(text: SourceText, classifiedSpan: ClassifiedSpan) : ClassifiedSpan = + let result = new List() + this.ClassifySourceTextAsync(text, classifiedSpan.TextSpan, result, CancellationToken.None).Wait() + if result.Any() then result.First() else new ClassifiedSpan(ClassificationTypeNames.WhiteSpace, classifiedSpan.TextSpan) + + member this.ClassifyDocumentAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = + let sourceText = document.GetTextAsync(cancellationToken).Result + // TODO: add types colorization if available from intellisense + this.ClassifySourceTextAsync(sourceText, textSpan, result, cancellationToken) + + member this.ClassifySourceTextAsync(text: SourceText, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = + Task.Run(fun () -> + let startLineNumber = text.Lines.GetLineFromPosition(textSpan.Start).LineNumber + let endLineNumber = text.Lines.GetLineFromPosition(textSpan.End).LineNumber + + for i = startLineNumber to endLineNumber - 1 do + cancellationToken.ThrowIfCancellationRequested() + + let currentLine = text.Lines.Item(i) + let previousLexState = ref(if i = 0 then 0L else lineEndingStates.GetOrAdd(i - 1, 0L)) + let tokens = this.ClassifySourceLineAsync(currentLine, previousLexState) + + result.AddRange(tokens) + lineEndingStates.[i] <- previousLexState.Value + ) + + member this.ClassifySourceLineAsync(textLine: TextLine, previousLexState: Ref): List = + let lineTokenizer = sourceTokenizer.CreateLineTokenizer(textLine.Text.ToString(textLine.Span)) + let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.WhiteSpace + + let scanNextToken() = + let tokenInfoOption, currentLexState = lineTokenizer.ScanToken(previousLexState.Value) + previousLexState := currentLexState + match tokenInfoOption with + | None -> false + | Some(tokenInfo) -> + let classificationType = this.CompilerTokenToRoslynToken(tokenInfo.ColorClass) + for i = tokenInfo.LeftColumn to tokenInfo.RightColumn do + Array.set colorMap i classificationType + true + + while scanNextToken() do () + + let mutable startPosition = 0 + let mutable endPosition = startPosition + let result = new List() + + while startPosition < colorMap.Length do + let classificationType = colorMap.[startPosition] + endPosition <- startPosition + while endPosition < colorMap.Length && classificationType = colorMap.[endPosition] do + endPosition <- endPosition + 1 + let textSpan = new TextSpan(textLine.Start + startPosition, endPosition - startPosition + 1) + result.Add(new ClassifiedSpan(classificationType, textSpan)) + startPosition <- endPosition + + result + + member this.CompilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) : string = + match colorKind with + | FSharpTokenColorKind.Comment -> ClassificationTypeNames.Comment + | FSharpTokenColorKind.Identifier -> ClassificationTypeNames.Identifier + | FSharpTokenColorKind.Keyword -> ClassificationTypeNames.Keyword + | FSharpTokenColorKind.String -> ClassificationTypeNames.StringLiteral + | FSharpTokenColorKind.Text -> ClassificationTypeNames.Text + | FSharpTokenColorKind.UpperIdentifier -> ClassificationTypeNames.Identifier + | FSharpTokenColorKind.Number -> ClassificationTypeNames.NumericLiteral + | FSharpTokenColorKind.InactiveCode -> ClassificationTypeNames.ExcludedCode + | FSharpTokenColorKind.PreprocessorKeyword -> ClassificationTypeNames.PreprocessorKeyword + | FSharpTokenColorKind.Operator -> ClassificationTypeNames.Operator + | FSharpTokenColorKind.TypeName -> ClassificationTypeNames.ClassName + | FSharpTokenColorKind.Default | _ -> ClassificationTypeNames.Text From f588f87d255235aae66670d5215c341b52ee98a3 Mon Sep 17 00:00:00 2001 From: Omar Tawfik Date: Thu, 25 Feb 2016 15:19:59 -0800 Subject: [PATCH 3/9] Added Brace Matching --- .../src/FSharp.Editor/BraceCompletion.fs | 135 ------------------ .../src/FSharp.Editor/FSharp.Editor.fsproj | 9 +- .../FSharpBraceMatchingService.fs | 79 ++++++++++ .../src/FSharp.Editor/TokenContext.fs | 29 ---- 4 files changed, 83 insertions(+), 169 deletions(-) delete mode 100644 vsintegration/src/FSharp.Editor/BraceCompletion.fs create mode 100644 vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs delete mode 100644 vsintegration/src/FSharp.Editor/TokenContext.fs diff --git a/vsintegration/src/FSharp.Editor/BraceCompletion.fs b/vsintegration/src/FSharp.Editor/BraceCompletion.fs deleted file mode 100644 index 4dff3674067..00000000000 --- a/vsintegration/src/FSharp.Editor/BraceCompletion.fs +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -namespace Microsoft.VisualStudio.FSharp.Editor - -open System -open System.ComponentModel.Composition -open Microsoft.VisualStudio -open Microsoft.VisualStudio.FSharp.LanguageService -open Microsoft.VisualStudio.Editor -open Microsoft.VisualStudio.Shell -open Microsoft.VisualStudio.Shell.Interop -open Microsoft.VisualStudio.Text -open Microsoft.VisualStudio.Text.BraceCompletion -open Microsoft.VisualStudio.Text.Editor -open Microsoft.VisualStudio.TextManager.Interop -open Microsoft.VisualStudio.Utilities - -/// F# brace completion context implementation -type internal CompletionContext(tokenContext : TokenContext, textManager : IVsTextManager) = - interface IBraceCompletionContext with - member this.Start(_) = - () - - /// Called when user hits Return while inside of a brace completion session. - member this.OnReturn(session) = - - let lp = [| LANGPREFERENCES(guidLang = Guid(FSharpCommonConstants.languageServiceGuidString)) |] - ErrorHandler.ThrowOnFailure(textManager.GetUserPreferences(null, null, lp, null)) |> ignore - - // if smart indent is not enabled, or we are in a string, don't do any special formatting - if (lp.[0].IndentStyle <> vsIndentStyle.vsIndentStyleSmart || session.OpeningBrace = '"') then - () - else - // within other braces: - // leave the opening brace where it is - // put the caret one line down, indented once from the previous line - // put the closing brace one line below that, justified with the original line - // let q = query { - // $caret$ - // } - - let openingBraceLine = session.OpeningPoint.GetPoint(session.SubjectBuffer.CurrentSnapshot).GetContainingLine() - let existingIndent = TextHelper.physicalIndentStringOfLine session.TextView openingBraceLine - let caretPosition = session.TextView.Caret.Position.BufferPosition.Position - let singleIndent = TextHelper.singleIndentString session.TextView - - // we will already have "existing" indentation applied, justifying the caret with the previous line. - // insert an additional indent, a newline, then another buffer matching "existing" indentation - use edit = session.SubjectBuffer.CreateEdit() - if edit.Insert(caretPosition, String.Format("{0}{1}{2}", singleIndent, Environment.NewLine, existingIndent)) then - let newSnapshot = edit.Apply() - - // if edit failed to apply, snapshots will be the same - if newSnapshot = edit.Snapshot then - () - else - session.TextView.Caret.MoveTo( - SnapshotPoint(newSnapshot, caretPosition + singleIndent.Length).TranslateTo(session.TextView.TextSnapshot, PointTrackingMode.Positive) - ) |> ignore - - /// Called when user types the (potential) closing brace for a session. - /// Return false if we don't think user is really completing the session. E.g. escaped \" should not close a string session - member this.AllowOverType(session) = - let caretPosition = session.TextView.Caret.Position.BufferPosition.Position - let line = session.TextView.TextSnapshot.GetLineFromPosition(caretPosition) - let lineNum = line.LineNumber - let braceColumn = caretPosition - line.Start.Position - - // test what state we'd be in if the closing brace and a space were typed - let tokenInfo = tokenContext.GetContextAt(session.TextView, lineNum, braceColumn + 1, (string session.ClosingBrace + " "), braceColumn) - - match tokenInfo.Type with - | TokenType.Comment - | TokenType.Unknown - | TokenType.String -> false - | _ -> true - - member this.Finish(_) = - () - -[)>] -[] -[] -[] -[] -[] -type BraceCompletionContextProvider [] (serviceProvider : SVsServiceProvider, adapterService : IVsEditorAdaptersFactoryService) = - - let tokenContext = TokenContext(serviceProvider, adapterService) - let textManager = serviceProvider.GetService(typeof) :?> IVsTextManager - - interface IBraceCompletionContextProvider with - - /// Called when user types a character matching a supported opening brace - member this.TryCreateContext(textView : ITextView , snap : SnapshotPoint, openingBrace : char, _ : char, completionContext : byref) = - if textView = null then raise (ArgumentException("textView")) else - - let line = snap.GetContainingLine() - let lineNum = line.LineNumber - let braceColumn = snap.Position - line.Start.Position - - // check what token context the user is at to decide if we should do brace completion - let info = tokenContext.GetContextAt(textView, lineNum, braceColumn, " ", braceColumn) - match info.Type with - | TokenType.String - | TokenType.Comment - | TokenType.Unknown -> - // don't initiate brace completion inside of strings, comments, or inactive code - completionContext <- null - false - | _ -> - // check if we might be in a char literal - let prevChar = - match snap.Position - 1 with - | p when p < 0 -> None - | p -> Some(snap.Snapshot.[p]) - - match prevChar with - | Some('\'') -> - // previous character was a ', but could this be a char literal? - // test the token context in the case that we added a closing ' after the brace character - let (insideCharLit, outsideCharLit) = - (tokenContext.GetContextAt(textView, lineNum, braceColumn, (string openingBrace) + "' ", braceColumn).Type, - tokenContext.GetContextAt(textView, lineNum, braceColumn + 2, (string openingBrace) + "' ", braceColumn).Type) - - match (insideCharLit, outsideCharLit) with - | (TokenType.String, t) when t <> TokenType.String -> - completionContext <- null - false - | _ -> - completionContext <- CompletionContext(tokenContext, textManager) - true - | _ -> - completionContext <- CompletionContext(tokenContext, textManager) - true \ No newline at end of file diff --git a/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj b/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj index 8f1d66ddb23..4a45ba272fc 100644 --- a/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj +++ b/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj @@ -22,15 +22,14 @@ $(OtherFlags) --warnon:1182 --subsystemversion:6.00 - - - false - - + + + false + diff --git a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs new file mode 100644 index 00000000000..e459306f066 --- /dev/null +++ b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Microsoft.VisualStudio.FSharp.Editor + +open System +open System.Composition +open System.Collections.Concurrent +open System.Collections.Generic +open System.Threading +open System.Threading.Tasks +open System.Linq + +open Microsoft.CodeAnalysis +open Microsoft.CodeAnalysis.Classification +open Microsoft.CodeAnalysis.Editor +open Microsoft.CodeAnalysis.Editor.Implementation.BraceMatching +open Microsoft.CodeAnalysis.Editor.Shared.Utilities +open Microsoft.CodeAnalysis.Host.Mef +open Microsoft.CodeAnalysis.Text + +open Microsoft.VisualStudio.FSharp.LanguageService +open Microsoft.VisualStudio.Text +open Microsoft.VisualStudio.Text.Tagging + +open Microsoft.FSharp.Compiler.Parser +open Microsoft.FSharp.Compiler.SourceCodeServices + +[] +type internal FSharpBraceMatchingService() = + + interface IBraceMatcher with + member this.FindBracesAsync(document: Document, position: int, cancellationToken: CancellationToken): Task> = + let computation = async { + let! text = document.GetTextAsync(cancellationToken) |> Async.AwaitTask + return this.FindBraces(text, position, cancellationToken) + } + Async.StartAsTask(computation, TaskCreationOptions.None, cancellationToken) + + member this.SupportedBraceTypes = [ + ('(', ')'); + ('<', '>'); + ('[', ']'); + ('{', '}'); + ] + + member this.FindBraces(text: SourceText, position: int, cancellationToken: CancellationToken) : Nullable = + let currentCharacter = text.[position] + + let proceedToLeft(i) = i - 1 + let proceedToRight(i) = i + 1 + + let afterEndOfString(i) = i >= text.Length + let beforeStartOfString(i) = i < 0 + + let pickBraceType(leftBrace, rightBrace) = + if currentCharacter = leftBrace then Some(proceedToRight, afterEndOfString, leftBrace, rightBrace) + else if currentCharacter = rightBrace then Some(proceedToLeft, beforeStartOfString, rightBrace, leftBrace) + else None + + match this.SupportedBraceTypes |> Seq.tryPick(pickBraceType) with + | None -> Nullable() + | Some(proceedFunc, stoppingCondition, matchedBrace, nonMatchedBrace) -> + let mutable currentPosition = proceedFunc position + let mutable result = Nullable() + let mutable braceDepth = 0 + + // TODO: ignore brace matching if part of a string literal or a comment + + while result.HasValue = false && stoppingCondition(currentPosition) = false do + cancellationToken.ThrowIfCancellationRequested() + if text.[currentPosition] = matchedBrace then + braceDepth <- braceDepth + 1 + else if text.[currentPosition] = nonMatchedBrace then + if braceDepth = 0 then + result <- Nullable(BraceMatchingResult(TextSpan(min position currentPosition, 1), TextSpan(max position currentPosition, 1))) + else + braceDepth <- braceDepth - 1 + currentPosition <- proceedFunc currentPosition + result diff --git a/vsintegration/src/FSharp.Editor/TokenContext.fs b/vsintegration/src/FSharp.Editor/TokenContext.fs deleted file mode 100644 index 8da5b61c4f8..00000000000 --- a/vsintegration/src/FSharp.Editor/TokenContext.fs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -namespace Microsoft.VisualStudio.FSharp.Editor - -open Microsoft.VisualStudio -open Microsoft.VisualStudio.Editor -open Microsoft.VisualStudio.TextManager.Interop -open Microsoft.VisualStudio.FSharp.LanguageService -open Microsoft.VisualStudio.Shell -open Microsoft.VisualStudio.Text.Editor - -/// helper class which provides token information -type internal TokenContext (serviceProvider : SVsServiceProvider, adapterService : IVsEditorAdaptersFactoryService) = - let fsLangService = serviceProvider.GetService(typeof) :?> FSharpLanguageService - - /// Returns token info for given position. - /// If trialString is provided, will return token info for given position, assuming trialString has been inserted at that position - member this.GetContextAt(textView : ITextView, lineNum : int, tokenColumn : int, trialString : string, trialStringInsertionColumn : int) = - let vsTextView = adapterService.GetViewAdapter(textView) - - let hr,buffer = vsTextView.GetBuffer() - ErrorHandler.ThrowOnFailure(hr) |> ignore - - let hr,colorizer = (fsLangService :> IVsLanguageInfo).GetColorizer(buffer) - ErrorHandler.ThrowOnFailure(hr) |> ignore - - let fsColorizer = colorizer :?> FSharpColorizer - - fsColorizer.GetTokenInfoAt(VsTextLines.TextColorState buffer, lineNum, tokenColumn, trialString, trialStringInsertionColumn) From 569db4c0a34c0989cb7f75bd7c52fcc610820c4f Mon Sep 17 00:00:00 2001 From: Omar Tawfik Date: Fri, 26 Feb 2016 13:10:13 -0800 Subject: [PATCH 4/9] Added ColorizationDataCache, and stopped brace matching inside comments/strings/excluded code --- .../FSharpBraceMatchingService.fs | 97 ++++++----- .../FSharpColorizationService.fs | 152 ++++++++++-------- 2 files changed, 148 insertions(+), 101 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs index e459306f066..df3992867ef 100644 --- a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs +++ b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs @@ -27,53 +27,76 @@ open Microsoft.FSharp.Compiler.SourceCodeServices [] type internal FSharpBraceMatchingService() = + + let SupportedBraceTypes = [ + ('(', ')'); + ('<', '>'); + ('[', ']'); + ('{', '}'); + ] + let IgnoredClassificationTypes = [ + ClassificationTypeNames.Comment; + ClassificationTypeNames.StringLiteral; + ClassificationTypeNames.ExcludedCode; + ] + interface IBraceMatcher with member this.FindBracesAsync(document: Document, position: int, cancellationToken: CancellationToken): Task> = let computation = async { let! text = document.GetTextAsync(cancellationToken) |> Async.AwaitTask - return this.FindBraces(text, position, cancellationToken) + return + try + this.FindBraces(text, position, cancellationToken) + with ex -> + Assert.Exception(ex) + reraise() } Async.StartAsTask(computation, TaskCreationOptions.None, cancellationToken) - - member this.SupportedBraceTypes = [ - ('(', ')'); - ('<', '>'); - ('[', ']'); - ('{', '}'); - ] - - member this.FindBraces(text: SourceText, position: int, cancellationToken: CancellationToken) : Nullable = - let currentCharacter = text.[position] + + member this.FindBraces(sourceText: SourceText, position: int, cancellationToken: CancellationToken) : Nullable = + if position < 0 || position >= sourceText.Length then + Nullable() + else + let classificationData = FSharpColorizationService.GetColorizationData(sourceText, cancellationToken) - let proceedToLeft(i) = i - 1 - let proceedToRight(i) = i + 1 + let shouldBeIgnored(characterPosition) = + match classificationData.GetClassifiedSpan(characterPosition) with + | None -> false + | Some(classifiedSpan) -> IgnoredClassificationTypes |> Seq.contains classifiedSpan.ClassificationType + + if shouldBeIgnored(position) then + Nullable() + else + let currentCharacter = sourceText.[position] - let afterEndOfString(i) = i >= text.Length - let beforeStartOfString(i) = i < 0 + let proceedToStartOfString(i) = i - 1 + let proceedToEndOfString(i) = i + 1 - let pickBraceType(leftBrace, rightBrace) = - if currentCharacter = leftBrace then Some(proceedToRight, afterEndOfString, leftBrace, rightBrace) - else if currentCharacter = rightBrace then Some(proceedToLeft, beforeStartOfString, rightBrace, leftBrace) - else None + let afterEndOfString(i) = i >= sourceText.Length + let beforeStartOfString(i) = i < 0 - match this.SupportedBraceTypes |> Seq.tryPick(pickBraceType) with - | None -> Nullable() - | Some(proceedFunc, stoppingCondition, matchedBrace, nonMatchedBrace) -> - let mutable currentPosition = proceedFunc position - let mutable result = Nullable() - let mutable braceDepth = 0 + let pickBraceType(leftBrace, rightBrace) = + if currentCharacter = leftBrace then Some(proceedToEndOfString, afterEndOfString, leftBrace, rightBrace) + else if currentCharacter = rightBrace then Some(proceedToStartOfString, beforeStartOfString, rightBrace, leftBrace) + else None - // TODO: ignore brace matching if part of a string literal or a comment + match SupportedBraceTypes |> Seq.tryPick(pickBraceType) with + | None -> Nullable() + | Some(proceedFunc, stoppingCondition, matchedBrace, nonMatchedBrace) -> + let mutable currentPosition = proceedFunc position + let mutable result = Nullable() + let mutable braceDepth = 0 - while result.HasValue = false && stoppingCondition(currentPosition) = false do - cancellationToken.ThrowIfCancellationRequested() - if text.[currentPosition] = matchedBrace then - braceDepth <- braceDepth + 1 - else if text.[currentPosition] = nonMatchedBrace then - if braceDepth = 0 then - result <- Nullable(BraceMatchingResult(TextSpan(min position currentPosition, 1), TextSpan(max position currentPosition, 1))) - else - braceDepth <- braceDepth - 1 - currentPosition <- proceedFunc currentPosition - result + while result.HasValue = false && stoppingCondition(currentPosition) = false do + cancellationToken.ThrowIfCancellationRequested() + if shouldBeIgnored(currentPosition) = false then + if sourceText.[currentPosition] = matchedBrace then + braceDepth <- braceDepth + 1 + else if sourceText.[currentPosition] = nonMatchedBrace then + if braceDepth = 0 then + result <- Nullable(BraceMatchingResult(TextSpan(min position currentPosition, 1), TextSpan(max position currentPosition, 1))) + else + braceDepth <- braceDepth - 1 + currentPosition <- proceedFunc currentPosition + result diff --git a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs index ea4f71fbec0..a9d75f3d876 100644 --- a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs +++ b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs @@ -6,9 +6,10 @@ open System open System.Composition open System.Collections.Concurrent open System.Collections.Generic +open System.Linq open System.Threading open System.Threading.Tasks -open System.Linq +open System.Runtime.CompilerServices open Microsoft.CodeAnalysis open Microsoft.CodeAnalysis.Classification @@ -24,93 +25,116 @@ open Microsoft.VisualStudio.Text.Tagging open Microsoft.FSharp.Compiler.SourceCodeServices +// TODO: add types colorization if available from intellisense + +type internal SourceTextColorizationData(classificationData: seq) = + member this.Tokens = classificationData |> Seq.toArray + member this.GetClassifiedSpan(position: int): Option = + let mutable left = 0 + let mutable right = this.Tokens.Length - 1 + let mutable result = None + while result.IsNone && right >= left do + let middle = (left + right) / 2 + let middleToken = this.Tokens.[middle] + if middleToken.TextSpan.End <= position then + left <- middle + 1 + else if middleToken.TextSpan.Start > position then + right <- middle - 1 + else + result <- Some(middleToken) + result + [, FSharpCommonConstants.FSharpLanguageName)>] type internal FSharpColorizationService() = - // Create expensive resources upfront - let sourceTokenizer = FSharpSourceTokenizer([], String.Empty) - let lineEndingStates = new ConcurrentDictionary() - interface IEditorClassificationService with member this.AddLexicalClassifications(text: SourceText, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - this.ClassifySourceTextAsync(text, textSpan, result, cancellationToken).Wait(cancellationToken) + FSharpColorizationService.ClassifySourceTextAsync(text, textSpan, result, cancellationToken).Wait(cancellationToken) member this.AddSyntacticClassificationsAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - this.ClassifyDocumentAsync(document, textSpan, result, cancellationToken) + let sourceText = document.GetTextAsync(cancellationToken).Result + FSharpColorizationService.ClassifySourceTextAsync(sourceText, textSpan, result, cancellationToken) member this.AddSemanticClassificationsAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - this.ClassifyDocumentAsync(document, textSpan, result, cancellationToken) + let sourceText = document.GetTextAsync(cancellationToken).Result + FSharpColorizationService.ClassifySourceTextAsync(sourceText, textSpan, result, cancellationToken) member this.AdjustStaleClassification(text: SourceText, classifiedSpan: ClassifiedSpan) : ClassifiedSpan = let result = new List() - this.ClassifySourceTextAsync(text, classifiedSpan.TextSpan, result, CancellationToken.None).Wait() - if result.Any() then result.First() else new ClassifiedSpan(ClassificationTypeNames.WhiteSpace, classifiedSpan.TextSpan) + FSharpColorizationService.ClassifySourceTextAsync(text, classifiedSpan.TextSpan, result, CancellationToken.None).Wait() + if result.Any() then + result.First() + else + new ClassifiedSpan(ClassificationTypeNames.WhiteSpace, classifiedSpan.TextSpan) - member this.ClassifyDocumentAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - let sourceText = document.GetTextAsync(cancellationToken).Result - // TODO: add types colorization if available from intellisense - this.ClassifySourceTextAsync(sourceText, textSpan, result, cancellationToken) - - member this.ClassifySourceTextAsync(text: SourceText, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - Task.Run(fun () -> - let startLineNumber = text.Lines.GetLineFromPosition(textSpan.Start).LineNumber - let endLineNumber = text.Lines.GetLineFromPosition(textSpan.End).LineNumber - for i = startLineNumber to endLineNumber - 1 do - cancellationToken.ThrowIfCancellationRequested() + static member ColorizationDataCache = ConditionalWeakTable() - let currentLine = text.Lines.Item(i) - let previousLexState = ref(if i = 0 then 0L else lineEndingStates.GetOrAdd(i - 1, 0L)) - let tokens = this.ClassifySourceLineAsync(currentLine, previousLexState) + static member GetColorizationData(sourceText: SourceText, cancellationToken: CancellationToken) : SourceTextColorizationData = + FSharpColorizationService.ColorizationDataCache.GetValue(sourceText, fun key -> FSharpColorizationService.ScanSourceText(key, cancellationToken)) - result.AddRange(tokens) - lineEndingStates.[i] <- previousLexState.Value + + static member ClassifySourceTextAsync(sourceText: SourceText, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = + Task.Run(fun () -> + try + let classificationData = FSharpColorizationService.GetColorizationData(sourceText, cancellationToken) + result.AddRange(classificationData.Tokens |> Seq.filter(fun token -> textSpan.Start <= token.TextSpan.Start && token.TextSpan.End <= textSpan.End)) + with ex -> + Assert.Exception(ex) + reraise() ) - member this.ClassifySourceLineAsync(textLine: TextLine, previousLexState: Ref): List = - let lineTokenizer = sourceTokenizer.CreateLineTokenizer(textLine.Text.ToString(textLine.Span)) - let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.WhiteSpace + static member ScanSourceText(sourceText: SourceText, cancellationToken: CancellationToken): SourceTextColorizationData = + let mutable runningLexState = ref(0L) + let result = new List() + let sourceTokenizer = FSharpSourceTokenizer([], String.Empty) - let scanNextToken() = - let tokenInfoOption, currentLexState = lineTokenizer.ScanToken(previousLexState.Value) - previousLexState := currentLexState + let compilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) = + match colorKind with + | FSharpTokenColorKind.Comment -> ClassificationTypeNames.Comment + | FSharpTokenColorKind.Identifier -> ClassificationTypeNames.Identifier + | FSharpTokenColorKind.Keyword -> ClassificationTypeNames.Keyword + | FSharpTokenColorKind.String -> ClassificationTypeNames.StringLiteral + | FSharpTokenColorKind.Text -> ClassificationTypeNames.Text + | FSharpTokenColorKind.UpperIdentifier -> ClassificationTypeNames.Identifier + | FSharpTokenColorKind.Number -> ClassificationTypeNames.NumericLiteral + | FSharpTokenColorKind.InactiveCode -> ClassificationTypeNames.ExcludedCode + | FSharpTokenColorKind.PreprocessorKeyword -> ClassificationTypeNames.PreprocessorKeyword + | FSharpTokenColorKind.Operator -> ClassificationTypeNames.Operator + | FSharpTokenColorKind.TypeName -> ClassificationTypeNames.ClassName + | FSharpTokenColorKind.Default | _ -> ClassificationTypeNames.Text + + let scanNextToken(lineTokenizer: FSharpLineTokenizer, colorMap: string[], lexState: Ref) = + let tokenInfoOption, currentLexState = lineTokenizer.ScanToken(lexState.Value) + lexState.Value <- currentLexState match tokenInfoOption with | None -> false | Some(tokenInfo) -> - let classificationType = this.CompilerTokenToRoslynToken(tokenInfo.ColorClass) + let classificationType = compilerTokenToRoslynToken(tokenInfo.ColorClass) for i = tokenInfo.LeftColumn to tokenInfo.RightColumn do Array.set colorMap i classificationType true - while scanNextToken() do () - - let mutable startPosition = 0 - let mutable endPosition = startPosition - let result = new List() - - while startPosition < colorMap.Length do - let classificationType = colorMap.[startPosition] - endPosition <- startPosition - while endPosition < colorMap.Length && classificationType = colorMap.[endPosition] do - endPosition <- endPosition + 1 - let textSpan = new TextSpan(textLine.Start + startPosition, endPosition - startPosition + 1) - result.Add(new ClassifiedSpan(classificationType, textSpan)) - startPosition <- endPosition - - result - - member this.CompilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) : string = - match colorKind with - | FSharpTokenColorKind.Comment -> ClassificationTypeNames.Comment - | FSharpTokenColorKind.Identifier -> ClassificationTypeNames.Identifier - | FSharpTokenColorKind.Keyword -> ClassificationTypeNames.Keyword - | FSharpTokenColorKind.String -> ClassificationTypeNames.StringLiteral - | FSharpTokenColorKind.Text -> ClassificationTypeNames.Text - | FSharpTokenColorKind.UpperIdentifier -> ClassificationTypeNames.Identifier - | FSharpTokenColorKind.Number -> ClassificationTypeNames.NumericLiteral - | FSharpTokenColorKind.InactiveCode -> ClassificationTypeNames.ExcludedCode - | FSharpTokenColorKind.PreprocessorKeyword -> ClassificationTypeNames.PreprocessorKeyword - | FSharpTokenColorKind.Operator -> ClassificationTypeNames.Operator - | FSharpTokenColorKind.TypeName -> ClassificationTypeNames.ClassName - | FSharpTokenColorKind.Default | _ -> ClassificationTypeNames.Text + let scanSourceLine(textLine: TextLine, lexState: Ref) = + let lineTokenizer = sourceTokenizer.CreateLineTokenizer(textLine.Text.ToString(textLine.Span)) + let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.WhiteSpace + while scanNextToken(lineTokenizer, colorMap, lexState) do () + + let mutable startPosition = 0 + let mutable endPosition = startPosition + while startPosition < colorMap.Length do + let classificationType = colorMap.[startPosition] + endPosition <- startPosition + while endPosition < colorMap.Length && classificationType = colorMap.[endPosition] do + endPosition <- endPosition + 1 + let textSpan = new TextSpan(textLine.Start + startPosition, endPosition - startPosition) + result.Add(new ClassifiedSpan(classificationType, textSpan)) + startPosition <- endPosition + + for i = 0 to sourceText.Lines.Count - 1 do + cancellationToken.ThrowIfCancellationRequested() + let currentLine = sourceText.Lines.Item(i) + scanSourceLine(currentLine, runningLexState) + + SourceTextColorizationData(result) From 5e75a0e9688c13bcd52e6bac4738216c83693af9 Mon Sep 17 00:00:00 2001 From: Omar Tawfik Date: Fri, 26 Feb 2016 15:56:12 -0800 Subject: [PATCH 5/9] Added placeholders for compiler defines --- .../FSharp.Editor/FSharpBraceMatchingService.fs | 2 +- .../FSharp.Editor/FSharpColorizationService.fs | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs index df3992867ef..0236d16a56c 100644 --- a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs +++ b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs @@ -58,7 +58,7 @@ type internal FSharpBraceMatchingService() = if position < 0 || position >= sourceText.Length then Nullable() else - let classificationData = FSharpColorizationService.GetColorizationData(sourceText, cancellationToken) + let classificationData = FSharpColorizationService.GetColorizationData(sourceText, [], cancellationToken) let shouldBeIgnored(characterPosition) = match classificationData.GetClassifiedSpan(characterPosition) with diff --git a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs index a9d75f3d876..31721c2fd18 100644 --- a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs +++ b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs @@ -26,6 +26,7 @@ open Microsoft.VisualStudio.Text.Tagging open Microsoft.FSharp.Compiler.SourceCodeServices // TODO: add types colorization if available from intellisense +// TODO: add defines flags if available from project sites and files type internal SourceTextColorizationData(classificationData: seq) = member this.Tokens = classificationData |> Seq.toArray @@ -69,26 +70,25 @@ type internal FSharpColorizationService() = new ClassifiedSpan(ClassificationTypeNames.WhiteSpace, classifiedSpan.TextSpan) - static member ColorizationDataCache = ConditionalWeakTable() + static member private ColorizationDataCache = ConditionalWeakTable() - static member GetColorizationData(sourceText: SourceText, cancellationToken: CancellationToken) : SourceTextColorizationData = - FSharpColorizationService.ColorizationDataCache.GetValue(sourceText, fun key -> FSharpColorizationService.ScanSourceText(key, cancellationToken)) + static member GetColorizationData(sourceText: SourceText, defines: string list, cancellationToken: CancellationToken) : SourceTextColorizationData = + FSharpColorizationService.ColorizationDataCache.GetValue(sourceText, fun key -> FSharpColorizationService.ScanSourceText(key, defines, cancellationToken)) - - static member ClassifySourceTextAsync(sourceText: SourceText, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = + static member private ClassifySourceTextAsync(sourceText: SourceText, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = Task.Run(fun () -> try - let classificationData = FSharpColorizationService.GetColorizationData(sourceText, cancellationToken) + let classificationData = FSharpColorizationService.GetColorizationData(sourceText, [], cancellationToken) result.AddRange(classificationData.Tokens |> Seq.filter(fun token -> textSpan.Start <= token.TextSpan.Start && token.TextSpan.End <= textSpan.End)) with ex -> Assert.Exception(ex) reraise() ) - static member ScanSourceText(sourceText: SourceText, cancellationToken: CancellationToken): SourceTextColorizationData = + static member private ScanSourceText(sourceText: SourceText, defines: string list, cancellationToken: CancellationToken): SourceTextColorizationData = let mutable runningLexState = ref(0L) let result = new List() - let sourceTokenizer = FSharpSourceTokenizer([], String.Empty) + let sourceTokenizer = FSharpSourceTokenizer(defines, String.Empty) let compilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) = match colorKind with From 25e9737146ee7a7b5c08fc84463ec4dceec2522c Mon Sep 17 00:00:00 2001 From: Omar Tawfik Date: Mon, 29 Feb 2016 11:52:08 -0800 Subject: [PATCH 6/9] Added Colorization/Brace Matching tests --- src/fsharp/lexhelp.fs | 15 +- .../src/FSharp.Editor/FSharp.Editor.fsproj | 1 + .../FSharpBraceMatchingService.fs | 28 +- .../FSharpColorizationService.fs | 22 +- .../src/FSharp.Editor/InternalsVisibleTo.fs | 10 + .../Tests.LanguageService.Colorizer.fs | 1091 ----------------- .../Tests.Roslyn.BraceMatchingService.fs | 119 ++ .../Tests.Roslyn.ColorizationService.fs | 1056 ++++++++++++++++ .../unittests/VisualFSharp.Unittests.fsproj | 26 +- 9 files changed, 1252 insertions(+), 1116 deletions(-) create mode 100644 vsintegration/src/FSharp.Editor/InternalsVisibleTo.fs delete mode 100644 vsintegration/tests/unittests/Tests.LanguageService.Colorizer.fs create mode 100644 vsintegration/tests/unittests/Tests.Roslyn.BraceMatchingService.fs create mode 100644 vsintegration/tests/unittests/Tests.Roslyn.ColorizationService.fs diff --git a/src/fsharp/lexhelp.fs b/src/fsharp/lexhelp.fs index 6aa2c1f18d7..9a6865cb7c6 100644 --- a/src/fsharp/lexhelp.fs +++ b/src/fsharp/lexhelp.fs @@ -2,6 +2,7 @@ module internal Microsoft.FSharp.Compiler.Lexhelp +open System open System.Text open Internal.Utilities open Internal.Utilities.Collections @@ -335,11 +336,15 @@ module Keywords = match s with | "__SOURCE_DIRECTORY__" -> let filename = fileOfFileIndex lexbuf.StartPos.FileIndex - let dirname = if filename = stdinMockFilename then - System.IO.Directory.GetCurrentDirectory() - else - filename |> FileSystem.GetFullPathShim (* asserts that path is already absolute *) - |> System.IO.Path.GetDirectoryName + let dirname = + if String.IsNullOrWhiteSpace(filename) then + String.Empty + else if filename = stdinMockFilename then + System.IO.Directory.GetCurrentDirectory() + else + filename + |> FileSystem.GetFullPathShim (* asserts that path is already absolute *) + |> System.IO.Path.GetDirectoryName KEYWORD_STRING dirname | "__SOURCE_FILE__" -> KEYWORD_STRING (System.IO.Path.GetFileName((fileOfFileIndex lexbuf.StartPos.FileIndex))) diff --git a/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj b/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj index 4a45ba272fc..186a42db4bd 100644 --- a/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj +++ b/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj @@ -22,6 +22,7 @@ $(OtherFlags) --warnon:1182 --subsystemversion:6.00 + diff --git a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs index 0236d16a56c..6e059a6f3c4 100644 --- a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs +++ b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs @@ -28,14 +28,14 @@ open Microsoft.FSharp.Compiler.SourceCodeServices [] type internal FSharpBraceMatchingService() = - let SupportedBraceTypes = [ + static member private SupportedBraceTypes = [ ('(', ')'); ('<', '>'); ('[', ']'); ('{', '}'); ] - let IgnoredClassificationTypes = [ + static member private IgnoredClassificationTypes = [ ClassificationTypeNames.Comment; ClassificationTypeNames.StringLiteral; ClassificationTypeNames.ExcludedCode; @@ -44,26 +44,38 @@ type internal FSharpBraceMatchingService() = interface IBraceMatcher with member this.FindBracesAsync(document: Document, position: int, cancellationToken: CancellationToken): Task> = let computation = async { - let! text = document.GetTextAsync(cancellationToken) |> Async.AwaitTask + let! sourceText = document.GetTextAsync(cancellationToken) |> Async.AwaitTask return try - this.FindBraces(text, position, cancellationToken) + FSharpBraceMatchingService.GetBraceMatchingResult(sourceText, document.Name, position, cancellationToken) with ex -> Assert.Exception(ex) reraise() } Async.StartAsTask(computation, TaskCreationOptions.None, cancellationToken) - member this.FindBraces(sourceText: SourceText, position: int, cancellationToken: CancellationToken) : Nullable = + static member FindMatchingBrace(sourceText: SourceText, fileName: string, position: int, cancellationToken: CancellationToken) : Option = + let braceMatchingResult = FSharpBraceMatchingService.GetBraceMatchingResult(sourceText, fileName, position, cancellationToken) + if braceMatchingResult.HasValue then + if braceMatchingResult.Value.LeftSpan.Start = position then + Some(braceMatchingResult.Value.RightSpan.Start) + else if braceMatchingResult.Value.RightSpan.Start = position then + Some(braceMatchingResult.Value.LeftSpan.Start) + else + None + else + None + + static member private GetBraceMatchingResult(sourceText: SourceText, fileName: string, position: int, cancellationToken: CancellationToken) : Nullable = if position < 0 || position >= sourceText.Length then Nullable() else - let classificationData = FSharpColorizationService.GetColorizationData(sourceText, [], cancellationToken) + let classificationData = FSharpColorizationService.GetColorizationData(sourceText, fileName, [], cancellationToken) let shouldBeIgnored(characterPosition) = match classificationData.GetClassifiedSpan(characterPosition) with | None -> false - | Some(classifiedSpan) -> IgnoredClassificationTypes |> Seq.contains classifiedSpan.ClassificationType + | Some(classifiedSpan) -> FSharpBraceMatchingService.IgnoredClassificationTypes |> Seq.contains classifiedSpan.ClassificationType if shouldBeIgnored(position) then Nullable() @@ -81,7 +93,7 @@ type internal FSharpBraceMatchingService() = else if currentCharacter = rightBrace then Some(proceedToStartOfString, beforeStartOfString, rightBrace, leftBrace) else None - match SupportedBraceTypes |> Seq.tryPick(pickBraceType) with + match FSharpBraceMatchingService.SupportedBraceTypes |> Seq.tryPick(pickBraceType) with | None -> Nullable() | Some(proceedFunc, stoppingCondition, matchedBrace, nonMatchedBrace) -> let mutable currentPosition = proceedFunc position diff --git a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs index 31721c2fd18..e4b0f347bf3 100644 --- a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs +++ b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs @@ -51,19 +51,19 @@ type internal FSharpColorizationService() = interface IEditorClassificationService with member this.AddLexicalClassifications(text: SourceText, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - FSharpColorizationService.ClassifySourceTextAsync(text, textSpan, result, cancellationToken).Wait(cancellationToken) + FSharpColorizationService.ClassifySourceTextAsync(text, String.Empty, textSpan, result, cancellationToken).Wait(cancellationToken) member this.AddSyntacticClassificationsAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = let sourceText = document.GetTextAsync(cancellationToken).Result - FSharpColorizationService.ClassifySourceTextAsync(sourceText, textSpan, result, cancellationToken) + FSharpColorizationService.ClassifySourceTextAsync(sourceText, document.Name, textSpan, result, cancellationToken) member this.AddSemanticClassificationsAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = let sourceText = document.GetTextAsync(cancellationToken).Result - FSharpColorizationService.ClassifySourceTextAsync(sourceText, textSpan, result, cancellationToken) + FSharpColorizationService.ClassifySourceTextAsync(sourceText, document.Name, textSpan, result, cancellationToken) member this.AdjustStaleClassification(text: SourceText, classifiedSpan: ClassifiedSpan) : ClassifiedSpan = let result = new List() - FSharpColorizationService.ClassifySourceTextAsync(text, classifiedSpan.TextSpan, result, CancellationToken.None).Wait() + FSharpColorizationService.ClassifySourceTextAsync(text, String.Empty, classifiedSpan.TextSpan, result, CancellationToken.None).Wait() if result.Any() then result.First() else @@ -72,23 +72,23 @@ type internal FSharpColorizationService() = static member private ColorizationDataCache = ConditionalWeakTable() - static member GetColorizationData(sourceText: SourceText, defines: string list, cancellationToken: CancellationToken) : SourceTextColorizationData = - FSharpColorizationService.ColorizationDataCache.GetValue(sourceText, fun key -> FSharpColorizationService.ScanSourceText(key, defines, cancellationToken)) + static member GetColorizationData(sourceText: SourceText, fileName: string, defines: string list, cancellationToken: CancellationToken) : SourceTextColorizationData = + FSharpColorizationService.ColorizationDataCache.GetValue(sourceText, fun key -> FSharpColorizationService.ScanSourceText(key, fileName, defines, cancellationToken)) - static member private ClassifySourceTextAsync(sourceText: SourceText, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = + static member private ClassifySourceTextAsync(sourceText: SourceText, fileName: string, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = Task.Run(fun () -> try - let classificationData = FSharpColorizationService.GetColorizationData(sourceText, [], cancellationToken) + let classificationData = FSharpColorizationService.GetColorizationData(sourceText, fileName, [], cancellationToken) result.AddRange(classificationData.Tokens |> Seq.filter(fun token -> textSpan.Start <= token.TextSpan.Start && token.TextSpan.End <= textSpan.End)) with ex -> Assert.Exception(ex) reraise() ) - static member private ScanSourceText(sourceText: SourceText, defines: string list, cancellationToken: CancellationToken): SourceTextColorizationData = + static member private ScanSourceText(sourceText: SourceText, fileName: string, defines: string list, cancellationToken: CancellationToken): SourceTextColorizationData = let mutable runningLexState = ref(0L) let result = new List() - let sourceTokenizer = FSharpSourceTokenizer(defines, String.Empty) + let sourceTokenizer = FSharpSourceTokenizer(defines, fileName) let compilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) = match colorKind with @@ -118,7 +118,7 @@ type internal FSharpColorizationService() = let scanSourceLine(textLine: TextLine, lexState: Ref) = let lineTokenizer = sourceTokenizer.CreateLineTokenizer(textLine.Text.ToString(textLine.Span)) - let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.WhiteSpace + let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.Text while scanNextToken(lineTokenizer, colorMap, lexState) do () let mutable startPosition = 0 diff --git a/vsintegration/src/FSharp.Editor/InternalsVisibleTo.fs b/vsintegration/src/FSharp.Editor/InternalsVisibleTo.fs new file mode 100644 index 00000000000..693a5eaa5b0 --- /dev/null +++ b/vsintegration/src/FSharp.Editor/InternalsVisibleTo.fs @@ -0,0 +1,10 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +namespace Microsoft.FSharp + +open System.Reflection + +[] + +do() + diff --git a/vsintegration/tests/unittests/Tests.LanguageService.Colorizer.fs b/vsintegration/tests/unittests/Tests.LanguageService.Colorizer.fs deleted file mode 100644 index 4a43f21dbbe..00000000000 --- a/vsintegration/tests/unittests/Tests.LanguageService.Colorizer.fs +++ /dev/null @@ -1,1091 +0,0 @@ -// Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. - -namespace Tests.LanguageService.Colorizer - -open System -open NUnit.Framework -open Salsa.Salsa -open Salsa.VsOpsUtils -open UnitTests.TestLib.Salsa -open UnitTests.TestLib.Utils -open UnitTests.TestLib.LanguageService -open UnitTests.TestLib.ProjectSystem - -// context msbuild -[] -type UsingMSBuild() = - inherit LanguageServiceBaseTests() - - //Marker At The End Helper Functions - member private this.VerifyColorizerAtEndOfMarker(fileContents : string, marker : string, tokenType : TokenType) = - let (solution, project, file) = this.CreateSingleFileProject(fileContents) - MoveCursorToEndOfMarker(file, marker) - AssertEqual(tokenType, GetTokenTypeAtCursor(file)) - - //Marker At The Start Helper Function - member private this.VerifyColorizerAtStartOfMarker(fileContents : string, marker : string, tokenType : TokenType) = - let (solution, project,file) = this.CreateSingleFileProject(fileContents) - MoveCursorToStartOfMarker(file, marker) - AssertEqual(tokenType, GetTokenTypeAtCursor(file)) - - [] - member this.``Comment.SingleLine``() = - this.VerifyColorizerAtEndOfMarker( - fileContents = """ - let simplefunction x y = x + y // Test1SimpleComment""", - marker = "// Test1", - tokenType = TokenType.Comment) - - [] - member this.``Conment.SingleLine.MultiConments``() = - this.VerifyColorizerAtEndOfMarker( - fileContents = """ - let x = // Test2SimpleComment // 1""", - marker = "// Test2", - tokenType = TokenType.Comment) - - [] - member this.``Comment.MultiLine.AfterAnExpression``() = - this.VerifyColorizerAtEndOfMarker( - fileContents = """ - let mutliLine x = 5(* Test1MultiLine - Test2MultiLine <@@asdf@@> - Test3MultiLine*) + 1(*Test4*)""", - marker = "Test1", - tokenType = TokenType.Comment) - - [] - member this.``Comment.MultiLine.WithLineBreakAndATab``() = - this.VerifyColorizerAtEndOfMarker( - fileContents = """ - let mutliLine x = 5(* Test1MultiLine - Test2MultiLine <@@asdf@@> - Test3MultiLine*) + 1(*Test4*) - """, - marker = "Test2", - tokenType = TokenType.Comment) - - [] - member this.``Comment.MultiLine.WithLineBreakAfterQuotExp``() = - this.VerifyColorizerAtEndOfMarker( - fileContents = """ - let mutliLine x = 5(* Test1MultiLine - Test2MultiLine <@@asdf@@> - Test3MultiLine*) + 1(*Test4*) - """, - marker = "Test3", - tokenType = TokenType.Comment) - - [] - member this.``Comment.MultiLine.AfterANumber``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let mutliLine x = 5(* Test1MultiLine - Test2MultiLine <@@asdf@@> - Test3MultiLine*) + 1(*Test4*) - """, - marker = "1(*Test4*)", - tokenType = TokenType.Number) - - [] - member this.``Comment.Nested.Nested01``() = - this.VerifyColorizerAtEndOfMarker( - fileContents = """ - (* L1Nesting - (* L2Nesting - (* L3 Nesting - let l3code = 3 - *) - let l2code = 2 - *) - let l1code = 1 - *) - let l0code = 0 - """, - marker = "let l3", - tokenType = TokenType.Comment) - - [] - member this.``Comment.Nested.Nested02``() = - this.VerifyColorizerAtEndOfMarker( - fileContents = """ - (* L1Nesting - (* L2Nesting - (* L3 Nesting - let l3code = 3 - *) - let l2code = 2 - *) - let l1code = 1 - *) - let l0code = 0 - """, - marker = "let l2", - tokenType = TokenType.Comment) - - [] - member this.``Comment.Nested.Nested03``() = - this.VerifyColorizerAtEndOfMarker( - fileContents = """ - (* L1Nesting - (* L2Nesting - (* L3 Nesting - let l3code = 3 - *) - let l2code = 2 - *) - let l1code = 1 - *) - let l0code = 0 - """, - marker = "let l1", - tokenType = TokenType.Comment) - - [] - member this.``Comment.Nested.IdentAfterNestedComments``() = - this.VerifyColorizerAtEndOfMarker( - fileContents = """ - (* L1Nesting - (* L2Nesting - (* L3 Nesting - let l3code = 3 - *) - let l2code = 2 - *) - let l1code = 1 - *) - let l0code = 0 - """, - marker = "let l0", - tokenType = TokenType.Identifier) - - [] - member this.``Comment.CommentInString``() = - this.VerifyColorizerAtEndOfMarker( - fileContents = """ - let commentsInString = "...(*test1_comment_in_string_literal*)..." - )""", - marker = "test1", - tokenType = TokenType.String) - - [] - member this.``Comment.StringInComment``() = - this.VerifyColorizerAtEndOfMarker( - fileContents = """ - (* - let commentsInString2 = "...*)test2_stringliteral_in_comment(*..." - *)""", - marker = "test2", - tokenType = TokenType.Comment) - - [] - member this.``Comment.Unterminated.KeywordBeforeComment``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - type IPeekPoke = interface(*ML Comment Start - abstract member Peek: unit -> int - abstract member Poke: int -> unit - end - """, - marker = "face(*ML Comment Start", - tokenType = TokenType.Keyword) - - [] - member this.``Comment.Unterminated.KeywordInComment``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - type IPeekPoke = interface(*ML Comment Start - abstract member Peek: unit -> int - abstract member Poke: int -> unit - end - - type wodget = class - val mutable state: int - interface IPeekPoke with(*Few Lines Later2*) - member x.Poke(n) = x.state <- x.state + n - member x.Peek() = x.state - end - end(*Few Lines Later3*)""", - marker = "with(*Few Lines Later2*)", - tokenType = TokenType.Comment) - - [] - member this.``Comment.Unterminated.NestedComments``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - type IPeekPoke = interface(*ML Comment Start - abstract member Peek: unit -> int - abstract member Poke: int -> unit - end - - type wodget = class - val mutable state: int - interface IPeekPoke with(*Few Lines Later2*) - member x.Poke(n) = x.state <- x.state + n - member x.Peek() = x.state - end - member x.HasBeenPoked = (x.state <> 0) - new() = { state = 0 } - end(*Few Lines Later3*)""", - marker = "nd(*Few Lines Later3*)", - tokenType = TokenType.Comment) - - [] - member this.``String.AtEnd``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let stringone = "simple string test"(*Simple String*) """, - marker = """est"(*Simple String*)""", tokenType = TokenType.String) - - [] - member this.``String.MultiLines``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let stringtwo = "simple test(*MultiLine - First*) - string test"(*MultiLine - Second*)""", - marker = "st(*MultiLine - First*)", - tokenType = TokenType.String) - - [] - member this.``String.MultiLines.LineBreak``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let stringtwo = "simple test(*MultiLine - First*) - string test"(*MultiLine - Second*) """, - marker = "\"(*MultiLine - Second*) ", - tokenType = TokenType.String) - - [] - member this.``String.Literal``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let stringthree = @"literal test"(*Literal String*)""", - marker = """st"(*Literal String*)""", - tokenType = TokenType.String) - - [] - member this.``ByteString.AtEnd``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let bytestringone = "abcdefg"B(*Byte String*)""", - marker = "B(*Byte String*)", tokenType = TokenType.String) - - [] - member this.``ByteString.MultiLines``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let bytestringtwo = "simple(*MultiLineB - First*) - string"B(*MultiLineB - Second*)""", - marker = """ing"B(*MultiLineB - Second*)""", - tokenType = TokenType.String) - - [] - member this.``ByteString.Literal``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let bytestringthree = @"literal"B(*Literal Byte*)""", - marker = """al"B(*Literal Byte*)""", - tokenType = TokenType.String) - - [] - member this.``EscapedIdentifier.word``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """let ``this is an escaped identifier 123ASDF@#$"`` = 4""", - marker = "`this", - tokenType = TokenType.Identifier) - - [] - member this.``EscapedIdentifier.SpecialChar``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """let ``this is an escaped identifier 123ASDF@#$"`` = 4""", - marker = "3ASDF@#", - tokenType = TokenType.Identifier) - - [] - member this.``EscapedIdentifier.EscapeChar``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """let ``this is an escaped identifier 123ASDF@#$"`` = 4""", - marker = "\"``", - tokenType = TokenType.Identifier) - - /// Regression for 3609 - Colorizer: __SOURCE__ and others colorized as a string - [] - member this.``PredefinedIdentifier.SOURCE_DIRECTORY``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let x = __SOURCE_DIRECTORY__(*Test1*)""", - marker = "__(*Test1*)", - tokenType = TokenType.Keyword) - - [] - member this.``PredefinedIdentifier.SOURCE_FILE``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let y = __SOURCE_FILE__(*Test2*))""", - marker = "__(*Test2*)", - tokenType = TokenType.Keyword) - - [] - member this.``PredefinedIdentifier.LINE``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let z = __LINE__(*Test3*)""", - marker = "__(*Test3*)", - tokenType = TokenType.Keyword) - - // Regression Test for FSB 3566, F# colorizer does not respect numbers - [] - member this.``Number.InAnExpression``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """let f x = x + 9""", - marker = "9", - tokenType = TokenType.Number) - - // Regression Test for FSB 1778 - Colorization seems to be confused after parsing a comment that contains a verbatim string that contains a \ - [] - member this.``Number.AfterCommentWithBackSlash``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """let f (* @"\\" *)x = x + 19(*Marker1*)""", - marker = "9(*Marker1*)", - tokenType = TokenType.Number) - - // Regression Test for FSharp1.0:2539 -- lexing @"" strings inside (* *) comments? - [] - member this.``Keyword.AfterCommentWithLexingStrings``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - (* - let x = @"\\" - *) - - let(*Marker1*) y = 1 - """, - marker = "t(*Marker1*)", - tokenType = TokenType.Keyword) - - // Regression Test for FSB 1380 - Language Service colorizes anything followed by a bang as a keyword - [] - member this.``Keyword.LetBang``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let seqExpr = - seq { - let! x = [1 .. 10](*Marker1*) - yield! x(*Marker2*) - do! - = ()(*Marker3*) - }""", - marker = "! x = [1 .. 10](*Marker1*)", - tokenType = TokenType.Keyword) - - [] - member this.``Keyword.Yield``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let seqExpr = - seq { - let! x = [1 .. 10](*Marker1*) - yield! x(*Marker2*) - do! - = ()(*Marker3*) - }""", - marker = "! x(*Marker2*)", - tokenType = TokenType.Keyword) - - [] - member this.``Keyword.Do``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let seqExpr = - seq { - let! x = [1 .. 10](*Marker1*) - yield! x(*Marker2*) - do! - = ()(*Marker3*) - }""", - marker = "! - = ()(*Marker3*)", - tokenType = TokenType.Keyword) - - [] - member this.``Keyword.Invalid.Bang``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let seqExpr = - seq { - foo! = true(*Marker1*) - }""", - marker = "! = true(*Marker1*)", - tokenType = TokenType.Identifier) - - [] - [] - [] - //This test case Verify that the color of const is the keyword color - member this.``TypeProvider.StaticParameters.Keyword.const``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - type foo = N1.T< const(*Marker1*) "Hello World",2>""", - marker = "t(*Marker1*)", - tokenType = TokenType.Keyword) - - // Regression test for FSB 3696 - Colorization doesn't treat #if/else/endif correctly when embedded in a string literal - [] - member this.``PreProcessor.InStringLiteral01``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - #if UNDEFINED - let x = "#elseMarker1" - let y = "#endifMarker2" - #else//Marker3 - let x = "#elseMarker4" - let y = "#endifMarker5" - #endif""", - marker = "eMarker1", - tokenType = TokenType.InactiveCode) - - [] - member this.``PreProcessor.InStringLiteral02``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - #if UNDEFINED - let x = "#elseMarker1" - let y = "#endifMarker2" - #else//Marker3 - let x = "#elseMarker4" - let y = "#endifMarker5" - #endif""", - marker = "fMarker2", - tokenType = TokenType.InactiveCode) - - [] - member this.``PreProcessor.ElseKeyword``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - #if UNDEFINED - let x = "#elseMarker1" - let y = "#endifMarker2" - #else//Marker3 - let x = "#elseMarker4" - let y = "#endifMarker5" - #endif""", - marker = "e//Marker3", - tokenType = TokenType.PreprocessorKeyword) - - [] - member this.``PreProcessor.InStringLiteral03``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - #if UNDEFINED - let x = "#elseMarker1" - let y = "#endifMarker2" - #else//Marker3 - let x = "#elseMarker4" - let y = "#endifMarker5" - #endif""", - marker = "eMarker4", - tokenType = TokenType.String) - - [] - member this.``PreProcessor.InStringLiteral04``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - #if UNDEFINED - let x = "#elseMarker1" - let y = "#endifMarker2" - #else//Marker3 - let x = "#elseMarker4" - let y = "#endifMarker5" - #endif""", - marker = "fMarker5", - tokenType = TokenType.String) - - // Regression test for FSHARP1.0:4279 - [] - member this.``Keyword.OCaml.asr``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let foo a = - match a with - | Some(asr, b) -> () - |_ -> ()""", - marker = "asr", - tokenType = TokenType.Keyword) - - [] - member this.``Keyword.OCaml.land``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let foo a = - match a with - | Some(land, b) -> () - |_ -> ()""", - marker = "land", - tokenType = TokenType.Keyword) - - [] - member this.``Keyword.OCaml.lor``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let foo a = - match a with - | Some(lor, b) -> () - |_ -> ()""", - marker = "lor", - tokenType = TokenType.Keyword) - - [] - member this.``Keyword.OCaml.lsl``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let foo a = - match a with - | Some(lsl, b) -> () - |_ -> ()""", - marker = "lsl", - tokenType = TokenType.Keyword) - - [] - member this.``Keyword.OCaml.lsr``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let foo a = - match a with - | Some(lsr, b) -> () - |_ -> ()""", - marker = "lsr", - tokenType = TokenType.Keyword) - - [] - member this.``Keyword.OCaml.lxor``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let foo a = - match a with - | Some(lxor, b) -> () - |_ -> ()""", - marker = "lxor", - tokenType = TokenType.Keyword) - - [] - member this.``Keyword.OCaml.mod``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let foo a = - match a with - | Some(mod, b) -> () - |_ -> ()""", - marker = "mod", - tokenType = TokenType.Keyword) - - [] - member this.``Keyword.OCaml.sig``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - let foo a = - match a with - | Some(sig, b) -> () - |_ -> ()""", - marker = "sig", - tokenType = TokenType.Keyword) - - [] - member this.InactiveCode() = - let fileContents = """ - #if UNDEFINED - (*Inactive Code1*)let notLegit1 x = x - #else - #if UNDEFINED - (*Inactive Code2*)let notLegit2 x = x - #else - #if UNDEFINED - (*Inactive Code3*)let notLegit3 x = x - #else - #if UNDEFINED - (*Inactive Code4*)let notLegit4 x = x - #else - #if UNDEFINED - (*Inactive Code5*)let notLegit5 x = x - #else - (*Active Code5*)let legitCode5 x = x - #endif - (*Active Code4*)let legitCode4 x = x - #endif - (*Active Code3*)let legitCode3 x = x - #endif - - (*Active Code2*)let legitCode2 x = x - #endif - (*Active Code1*)let legitCode1 x = x - #endif - - #if DEFINED - (*Active Code6*)let legitCode6 x = x - #if DEFINED - (*Active Code7*)let legitCode7 x = x - #else - (*Inactive Code7*)let notLegit7 x = x - #endif - #else - (*Inactive Code6*)let notLegit6 x = x - #endif - """ - - let (_solution, _project, file) = this.CreateSingleFileProject(fileContents, defines = ["DEFINED"]) - MoveCursorToEndOfMarker(file, "Active Code1*)le"); AssertEqual(TokenType.Keyword, GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file, "Active Code2*)le"); AssertEqual(TokenType.Keyword, GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file, "Active Code3*)le"); AssertEqual(TokenType.Keyword, GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file, "Active Code4*)le"); AssertEqual(TokenType.Keyword, GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file, "Active Code5*)le"); AssertEqual(TokenType.Keyword, GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file, "Active Code6*)le"); AssertEqual(TokenType.Keyword, GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file, "Active Code7*)le"); AssertEqual(TokenType.Keyword, GetTokenTypeAtCursor(file)) - - MoveCursorToEndOfMarker(file, "Inactive Code1*)le"); AssertEqual(TokenType.InactiveCode, GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file, "Inactive Code2*)le"); AssertEqual(TokenType.InactiveCode, GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file, "Inactive Code3*)le"); AssertEqual(TokenType.InactiveCode, GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file, "Inactive Code4*)le"); AssertEqual(TokenType.InactiveCode, GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file, "Inactive Code5*)le"); AssertEqual(TokenType.InactiveCode, GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file, "Inactive Code6*)le"); AssertEqual(TokenType.InactiveCode, GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file, "Inactive Code7*)le"); AssertEqual(TokenType.InactiveCode, GetTokenTypeAtCursor(file)) - - - - //ColorizerTest start - [] - member public this.``Regression.Bug2986``() = - let code = - [ - "#light" - "let x = 12" - "(*" - "blaa" - "blaa" - "blaa" - "blaa" - "blaa" - "blaa" - "blaa" - "*)" - "open System" - "open System.IO" - "open System.Text" - "open System.Security.Cryptography" - "let fold = List.fold" - "let linkMap f xs = List.concat (List.map f xs)" - "let argv = System.Environment.GetCommandLineArgs()" - "let buildConfiguration = argv.[argv.Length - 1]" - ] - let (_solution, _project, file) = this.CreateSingleFileProject(code) - - // Make sure things are colored right to start - MoveCursorToEndOfMarker(file,"open Sys") - AssertEqual(TokenType.Identifier,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file,"open System.Security.Crypto") - AssertEqual(TokenType.Identifier,GetTokenTypeAtCursor(file)) - - // Delete the chunk of comments. - ReplaceFileInMemory file - [ - "#light" - "let x = 12" - "open System" - "open System.IO" - "open System.Text" - "open System.Security.Cryptography" - "let fold = List.fold" - "let linkMap f xs = List.concat (List.map f xs)" - "let argv = System.Environment.GetCommandLineArgs()" - "let buildConfiguration = argv.[argv.Length - 1]" - ] - - // Reconfirm - MoveCursorToEndOfMarker(file,"open Sys") - AssertEqual(TokenType.Identifier,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file,"open System.Security.Crypto") - AssertEqual(TokenType.Identifier,GetTokenTypeAtCursor(file)) - - [] - member public this.``Colorizer.AtString``() = - let (_solution, _project, file) = this.CreateSingleFileProject("let s = @\"Bob\"") - // Check Bob - MoveCursorToEndOfMarker(file,"let s = @\"B") - AssertEqual(TokenType.String,GetTokenTypeAtCursor(file)) - - [] - member public this.``Regression.Bug4860``() = - let fileContents = " -let x = __SOURCE_DIRECTORY__(*Test1*) -let y = __SOURCE_FILE__(*Test2*) -let z = __LINE__(*Test3*) -" - - let (solution, project, file) = this.CreateSingleFileProject(fileContents) - - MoveCursorToStartOfMarker(file, "__(*Test1*)") - AssertEqual(TokenType.Keyword, GetTokenTypeAtCursor(file)) - - MoveCursorToStartOfMarker(file, "__(*Test2*)") - AssertEqual(TokenType.Keyword, GetTokenTypeAtCursor(file)) - - MoveCursorToStartOfMarker(file, "__(*Test3*)") - AssertEqual(TokenType.Keyword, GetTokenTypeAtCursor(file)) - - [] - member public this.``Number.Regression.Bug3566``() = - let otherNumbers = "let other = 0x4, 0b0100, 4L, 4UL, 4u, 4s, 4us, 4y, 4uy, 4.0, 4.0f, 4N, 4I, 1M, 123" - let code = - [ "let n = 123"; - "let l = [12..15]"; - "let l2 = [12 .. 15]"; - "let l3 = [ 12 .. 15 ]"; - "// comment1: 1234"; - "(* comment2: 1234 *)"; - otherNumbers; - ] - let (_solution, _project, file) = this.CreateSingleFileProject(code) - - // Check integer number - MoveCursorToEndOfMarker(file,"let n = 1") - AssertEqual(TokenType.Number,GetTokenTypeAtCursor(file)) - - // Check numbers in a range expression - MoveCursorToEndOfMarker(file,"let l = [1") - AssertEqual(TokenType.Number,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file,"let l = [12..1") - AssertEqual(TokenType.Number,GetTokenTypeAtCursor(file)) - - MoveCursorToEndOfMarker(file,"let l2 = [1") - AssertEqual(TokenType.Number,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file,"let l2 = [12 .. 1") - AssertEqual(TokenType.Number,GetTokenTypeAtCursor(file)) - - MoveCursorToEndOfMarker(file,"let l3 = [ 1") - AssertEqual(TokenType.Number,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file,"let l3 = [ 12 .. 1") - AssertEqual(TokenType.Number,GetTokenTypeAtCursor(file)) - - // Check other numeric formats - let mutable index = otherNumbers.IndexOf(",") - while(index <> -1) do - let substr = otherNumbers.Substring(0, index - 1) // -1 to move into the number - MoveCursorToEndOfMarker(file, substr) - AssertEqual(TokenType.Number, GetTokenTypeAtCursor(file)) - index <- otherNumbers.IndexOf(",", index + 1) - - // Check that numbers in comments are not colored as numbers - MoveCursorToEndOfMarker(file,"// comment1: 12") - AssertEqual(TokenType.Comment,GetTokenTypeAtCursor(file)) - - MoveCursorToEndOfMarker(file,"(* comment2: 12") - AssertEqual(TokenType.Comment,GetTokenTypeAtCursor(file)) - - - /// FEATURE: Hash commands in .fsx files are colorized in PreprocessorKeyword color - [] - member public this.``Preprocessor.InFsxFile``() = - let code = - [ - "#reference @\"\"" - "#load @\"\"" - "#I <--hash I" - "#time @\"\"" - " #reference @\"\"" - " #load @\"\"" - " #I <--spaces then hash I" - " #time @\"\"" - ] - let (_, _, file) = this.CreateSingleFileProject(code, fileKind = SourceFileKind.FSX) - - MoveCursorToEndOfMarker(file,"#ref") - AssertEqual(TokenType.PreprocessorKeyword ,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file,"#loa") - AssertEqual(TokenType.PreprocessorKeyword ,GetTokenTypeAtCursor(file)) - MoveCursorToStartOfMarker(file,"I <--hash I") - AssertEqual(TokenType.PreprocessorKeyword ,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file,"#ti") - AssertEqual(TokenType.PreprocessorKeyword ,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file," #ref") - AssertEqual(TokenType.PreprocessorKeyword ,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file," #loa") - AssertEqual(TokenType.PreprocessorKeyword ,GetTokenTypeAtCursor(file)) - MoveCursorToStartOfMarker(file,"I <--spaces then hash I") - AssertEqual(TokenType.PreprocessorKeyword ,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file," #ti") - AssertEqual(TokenType.PreprocessorKeyword ,GetTokenTypeAtCursor(file)) - - - /// FEATURE: Script-specific hash commands do not show up in blue in .fs files. - [] - member public this.``Preprocessor.InFsFile``() = - let code = - [ - "#reference @\"\"" - "#load @\"\"" - "#I <--hash I" - "#time @\"\"" - " #reference @\"\"" - " #load @\"\"" - " #I <--spaces then hash I" - " #time @\"\"" - ] - let (_, _, file) = this.CreateSingleFileProject(code) - - MoveCursorToEndOfMarker(file,"#ref") - AssertEqual(TokenType.Text,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file,"#loa") - AssertEqual(TokenType.Text ,GetTokenTypeAtCursor(file)) - MoveCursorToStartOfMarker(file,"I <--hash I") - AssertEqual(TokenType.Text ,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file,"#ti") - AssertEqual(TokenType.Text ,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file," #ref") - AssertEqual(TokenType.Text ,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file," #loa") - AssertEqual(TokenType.Text ,GetTokenTypeAtCursor(file)) - MoveCursorToStartOfMarker(file,"I <--spaces then hash I") - AssertEqual(TokenType.Text ,GetTokenTypeAtCursor(file)) - MoveCursorToEndOfMarker(file," #ti") - AssertEqual(TokenType.Text ,GetTokenTypeAtCursor(file)) - - /// FEATURE: Nested (* *) comments are allowed and will be colorized with CommentColor. Only the final *) causes the comment to close. - [] - member public this.``Comment.AfterCommentBlock``() = - let code = - ["(*Bob*)type Bob() = class end" - "(*" - "(*" - "(*Alice*)type Alice() = class end" - "*)" - "*)" - "(*Charles*)type Charles() = class end"] - let (_, _, file) = this.CreateSingleFileProject(code) - // Check Bob - MoveCursorToEndOfMarker(file,"(*Bob*)t") - AssertEqual(TokenType.Keyword,GetTokenTypeAtCursor(file)) - - // Check Alice - MoveCursorToEndOfMarker(file,"(*Alice*)t") - AssertEqual(TokenType.Comment,GetTokenTypeAtCursor(file)) - - // Check Charles - MoveCursorToEndOfMarker(file,"(*Charles*)t") - AssertEqual(TokenType.Keyword,GetTokenTypeAtCursor(file)) - - /// BUG: The comment used to be colored in black. - [] - member public this.``Regression.Bug1596``() = - let code = [" let 2d (* Identifiers cannot start with numbers *)"] - let (_, _, file) = this.CreateSingleFileProject(code) - - // Check Bob - MoveCursorToEndOfMarker(file,"let 2d (* Ide") - AssertEqual(TokenType.Comment,GetTokenTypeAtCursor(file)) - - - /// FEATURE: Code inside #if\#else\#endif blocks is colored with InactiveCodeColor depending on defines. This works for nested #if blocks as well. - [] - member public this.``Preprocessor.AfterPreprocessorBlock``() = - let code = - ["(*Bob*)type Bob() = class end" - "#if UNDEFINED" - " #if UNDEFINED" - " (*Alice*)type Alice() = class end" - " #else" - " (*Tom*)type Tom() = class end" - " #endif" - "#else" - " #if UNDEFINED" - " (*Maurice*)type Maurice() = class end" - " #else" - " (*Larry*)type Larry() = class end" - " #endif" - "#endif" - "(*Charles*)type Charles() = class end"] - let (_, _, file) = this.CreateSingleFileProject(code) - - let check marker token = - MoveCursorToEndOfMarker(file,marker) - AssertEqual(token,GetTokenTypeAtCursor(file)) - - check "(*Bob*)t" TokenType.Keyword - check "(*Alice*)t" TokenType.InactiveCode - check "(*Tom*)t" TokenType.InactiveCode - check "(*Maurice*)t" TokenType.InactiveCode - check "(*Larry*)t" TokenType.Keyword - check "(*Charles*)t" TokenType.Keyword - - // Wrong "#else" in "#if" should be ignored - [] - member public this.``Preprocessor.InvalidElseDirectiveIgnored``() = - let code = - ["#if UNDEFINED" - " (*Alice*)type Alice() = class end" - "(**) #else" - " (*Larry*)type Larry() = class end" - "#endif"] - let (_, _, file) = this.CreateSingleFileProject(code) - let check marker token = - MoveCursorToEndOfMarker(file,marker) - AssertEqual(token,GetTokenTypeAtCursor(file)) - - check "(*Alice*)t" TokenType.InactiveCode - check "(*Larry*)t" TokenType.InactiveCode - - /// FEATURE: Code inside #if\#else\#endif blocks is colored with InactiveCodeColor depending on defines. This works for nested #if blocks as well. - [] - member public this.``Preprocessor.AfterPreprocessorBlockWithDefines``() = - let code = - ["(*Bob*)type Bob() = class end" - "#if UNDEFINED" - " #if UNDEFINED" - " (*Alice*)type Alice() = class end" - " #else" - " (*Tom*)type Tom() = class end" - " #endif" - "#else" - " #if UNDEFINED" - " (*Maurice*)type Maurice() = class end" - " #else" - " (*Larry*)type Larry() = class end" - " #endif" - "#endif" - "(*Charles*)type Charles() = class end"] - let (_, _, file) = this.CreateSingleFileProject(code, defines = ["FOO";"UNDEFINED"]) - - let check marker token = - MoveCursorToEndOfMarker(file,marker) - AssertEqual(token,GetTokenTypeAtCursor(file)) - - check "(*Bob*)t" TokenType.Keyword - check "(*Alice*)t" TokenType.Keyword - check "(*Tom*)t" TokenType.InactiveCode - check "(*Maurice*)t" TokenType.InactiveCode - check "(*Larry*)t" TokenType.InactiveCode - check "(*Charles*)t" TokenType.Keyword - - /// FEATURE: Preprocessor keywords #light\#if\#else\#endif are colored with the PreprocessorKeyword color. - /// FEATURE: All code in the inactive side of #if\#else\#endif is colored with with InactiveCode color. - [] - member public this.``Preprocessor.Keywords``() = - let code = - ["#light (*Light*)" - " #if UNDEFINED //(*If*)" - " let x = 1(*Inactive*)" - " #else //(*Else*)" - " let(*Active*) x = 1" - " #endif //(*Endif*)"] - let (_, _, file) = this.CreateSingleFileProject(code) - - let check marker token = - MoveCursorToStartOfMarker(file,marker) - AssertEqual(token,GetTokenTypeAtCursor(file)) - - check "light (*Light*)" TokenType.PreprocessorKeyword - check "(*Inactive*)" TokenType.InactiveCode - check "if UNDEFINED //(*If*)" TokenType.PreprocessorKeyword - check "FINED //(*If*)" TokenType.Identifier - check "(*If*)" TokenType.Comment - check "else //(*Else*)" TokenType.PreprocessorKeyword - check "t(*Active*)" TokenType.Keyword - check "endif //(*Endif*)" TokenType.PreprocessorKeyword - check "(*If*)" TokenType.Comment - check "(*Else*)" TokenType.Comment - check "(*Endif*)" TokenType.Comment - - /// FEATURE: Preprocessor extended grammar basic check. - /// FEATURE: More extensive grammar test is done in compiler unit tests - [] - member public this.``Preprocessor.ExtendedIfGrammar.Basic01``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - #if UNDEFINED || !UNDEFINED // Extended #if - let x = "activeCode" - #else - let x = "inactiveCode" - #endif - """, - marker = "activeCode", - tokenType = TokenType.String) - - [] - member public this.``Preprocessor.ExtendedIfGrammar.Basic02``() = - this.VerifyColorizerAtStartOfMarker( - fileContents = """ - #if UNDEFINED || !UNDEFINED // Extended #if - let x = "activeCode" - #else - let x = "inactiveCode" - #endif - """, - marker = "inactiveCode", - tokenType = TokenType.InactiveCode) - - /// #else / #endif in multiline strings is ignored - [] - member public this.``Preprocessor.DirectivesInString``() = - let code = - ["#light" - "" - "#if DEFINED" - "let s = \"" - "#else" - "\"" - "let testme = 1" - "#endif"] - let (_, _, file) = this.CreateSingleFileProject(code, defines = ["DEFINED"]) - MoveCursorToStartOfMarker(file,"let testme") - AssertEqual(TokenType.Keyword,GetTokenTypeAtCursor(file)) - - /// Bug 2076 - String literals were causing the endif stack information to be discarded - [] - member public this.``Preprocessor.KeywordsWithStrings``() = - let code = - ["#light (*Light*)" - "let x1 = \"string1\"" - "#if UNDEFINED //(*If*)" - "let x2 = \"string2\"" - "#else //(*Else*)" - "let x3 = \"string3\"" - "#endif //(*Endif*)" - "let x4 = \"string4\""] - let (_, _, file) = this.CreateSingleFileProject(code) - - let check marker token = - MoveCursorToStartOfMarker(file,marker) - AssertEqual(token,GetTokenTypeAtCursor(file)) - check "if UNDEFINED //(*If*)" TokenType.PreprocessorKeyword - check "else //(*Else*)" TokenType.PreprocessorKeyword - check "endif //(*Endif*)" TokenType.PreprocessorKeyword - - [] - member public this.``Comment.VerbatimStringInComment.Bug1778``() = - let code = - ["#light" - "(* @\"\\\" *) let a = 0"] - let (_, _, file) = this.CreateSingleFileProject(code) - - MoveCursorToStartOfMarker(file, "le") - AssertEqual(TokenType.Keyword ,GetTokenTypeAtCursor(file)) - - [] - member public this.``Preprocessor.KeywordsWrongIf.Bug1577``() = - let code = - ["#if !!!!!!!!!!!!!!!COMPILED" - "#endif"] - let (_, _, file) = this.CreateSingleFileProject(code) - MoveCursorToStartOfMarker(file, "!!COMPILED") - AssertEqual(TokenType.Identifier, GetTokenTypeAtCursor(file)) - - - // This was an off-by-one bug in the replacement Colorizer - [] - member public this.``Keyword.LastCharacterOfKeyword``() = - let code = ["(*Bob*)type Bob() = int"] - let (_, _, file) = this.CreateSingleFileProject(code) - - // Check Bob - MoveCursorToEndOfMarker(file,"(*Bob*)typ") - AssertEqual(TokenType.Keyword,GetTokenTypeAtCursor(file)) - -// Context project system -[] -type UsingProjectSystem() = - inherit UsingMSBuild(VsOpts = LanguageServiceExtension.ProjectSystemTestFlavour) - - \ No newline at end of file diff --git a/vsintegration/tests/unittests/Tests.Roslyn.BraceMatchingService.fs b/vsintegration/tests/unittests/Tests.Roslyn.BraceMatchingService.fs new file mode 100644 index 00000000000..8a1e7a9f03d --- /dev/null +++ b/vsintegration/tests/unittests/Tests.Roslyn.BraceMatchingService.fs @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +namespace Microsoft.VisualStudio.FSharp.Editor.Tests.Roslyn + +open System +open System.Threading + +open NUnit.Framework + +open Microsoft.CodeAnalysis.Classification +open Microsoft.CodeAnalysis.Editor +open Microsoft.CodeAnalysis.Text +open Microsoft.VisualStudio.FSharp.Editor + +[] +type BraceMatchingServiceTests() = + + member private this.VerifyNoBraceMatch(fileContents: string, marker: string) = + let markerPosition = fileContents.IndexOf(marker) + Assert.IsTrue(markerPosition >= 0, "Cannot find marker '{0}' in file contents", marker) + + match FSharpBraceMatchingService.FindMatchingBrace(SourceText.From(fileContents), "test.fs", markerPosition, CancellationToken.None) with + | None -> () + | Some(foundMatch) -> Assert.Fail("Found match for brace at position '{0}'", foundMatch) + + member private this.VerifyBraceMatch(fileContents: string, startMarker: string, endMarker: string) = + let startMarkerPosition = fileContents.IndexOf(startMarker) + Assert.IsTrue(startMarkerPosition >= 0, "Cannot find start marker '{0}' in file contents", startMarkerPosition) + let endMarkerPosition = fileContents.IndexOf(endMarker) + Assert.IsTrue(endMarkerPosition >= 0, "Cannot find end marker '{0}' in file contents", endMarkerPosition) + + match FSharpBraceMatchingService.FindMatchingBrace(SourceText.From(fileContents), "test.fs", startMarkerPosition, CancellationToken.None) with + | None -> Assert.Fail("Didn't find a match for brace at position '{0}", startMarkerPosition) + | Some(foundMatch) -> Assert.AreEqual(endMarkerPosition, foundMatch, "Found match at incorrect position") + + + // Starting Brace + [] + [] + [] + [] + [marker5")>] + // Ending Brace + [] + [] + [] + [] + [marker5", "] + member this.NestedBrackets(startMarker: string, endMarker: string) = + let code = " + (marker1 + {marker2 + (marker3 + [marker4 + marker5 + ]marker4 + )marker3 + }marker2 + )marker1" + this.VerifyBraceMatch(code, startMarker, endMarker) + + [] + member this.BracketInExpression() = + this.VerifyBraceMatch("let x = (3*5)-1", "(3*", ")-1") + + [] + [] + member this.BraceInMultiLineCommentShouldnotBeMatched(startMarker: string) = + let code = " + let x = 3 + (* This [start + is a multiline + comment ]end + *) + printf \"%d\" x" + this.VerifyNoBraceMatch(code, startMarker) + + [] + member this.BraceEncapsulatingACommentShouldBeMatched() = + let code = " + let x = 3 + (start + (* this is a comment *) + )end" + this.VerifyBraceMatch(code, "(start", ")end") + + [] + [] + [] + [] + member this.BraceStartingOrEndingInCommentShouldnotBeMatched(startMarker: string) = + let code = " + let x = 123 + (endsInComment + (* )endsInComment startsInComment" + this.VerifyNoBraceMatch(code, startMarker) + + [] + [] + [] + [] + member this.BraceStartingOrEndingInDisabledCodeShouldnotBeMatched(startMarker: string) = + let code = " + let x = 123 + (endsInDisabledCode + #if UNDEFINED + )endsInDisabledCode ] + [] + [] + [] + member this.BraceStartingOrEndingInStringShouldnotBeMatched(startMarker: string) = + let code = " + let x = \"stringValue\" + (endsInString + + \" )endsInString ] +type ColorizationServiceTests() = + + member private this.VerifyColorizerAtStartOfMarker(fileContents: string, marker: string, defines: string list, classificationType: string, ?isScriptFile: bool) = + let fileName = if isScriptFile.IsSome && isScriptFile.Value then "test.fsx" else "test.fs" + let colorizationData = FSharpColorizationService.GetColorizationData(SourceText.From(fileContents), fileName, defines, CancellationToken.None) + let markerPosition = fileContents.IndexOf(marker) + Assert.IsTrue(markerPosition >= 0, "Cannot find marker '{0}' in file contents", marker) + + match colorizationData.GetClassifiedSpan(markerPosition) with + | None -> Assert.Fail("Cannot find colorization data for start of marker") + | Some(classifiedSpan) -> Assert.AreEqual(classificationType, classifiedSpan.ClassificationType, "Classification data doesn't match for start of marker") + + member private this.VerifyColorizerAtEndOfMarker(fileContents : string, marker : string, defines: string list, classificationType: string, ?isScriptFile: bool) = + let fileName = if isScriptFile.IsSome && isScriptFile.Value then "test.fsx" else "test.fs" + let colorizationData = FSharpColorizationService.GetColorizationData(SourceText.From(fileContents), fileName, defines, CancellationToken.None) + let markerPosition = fileContents.IndexOf(marker) + Assert.IsTrue(markerPosition >= 0, "Cannot find marker '{0}' in file contents", marker) + + match colorizationData.GetClassifiedSpan(markerPosition + marker.Length - 1) with + | None -> Assert.Fail("Cannot find colorization data for end of marker") + | Some(classifiedSpan) -> Assert.AreEqual(classificationType, classifiedSpan.ClassificationType, "Classification data doesn't match for end of marker") + + + [] + member this.Comment_SingleLine() = + this.VerifyColorizerAtEndOfMarker( + fileContents = """ + let simplefunction x y = x + y // Test1SimpleComment""", + marker = "// Test1", + defines = [], + classificationType = ClassificationTypeNames.Comment) + + [] + member this.Conment_SingleLine_MultiConments() = + this.VerifyColorizerAtEndOfMarker( + fileContents = """ + let x = // Test2SimpleComment // 1""", + marker = "// Test2", + defines = [], + classificationType = ClassificationTypeNames.Comment) + + [] + member this.Comment_MultiLine_AfterAnExpression() = + this.VerifyColorizerAtEndOfMarker( + fileContents = """ + let mutliLine x = 5(* Test1MultiLine + Test2MultiLine <@@asdf@@> + Test3MultiLine*) + 1(*Test4*)""", + marker = "Test1", + defines = [], + classificationType = ClassificationTypeNames.Comment) + + [] + member this.Comment_MultiLine_WithLineBreakAndATab() = + this.VerifyColorizerAtEndOfMarker( + fileContents = """ + let mutliLine x = 5(* Test1MultiLine + Test2MultiLine <@@asdf@@> + Test3MultiLine*) + 1(*Test4*) + """, + marker = "Test2", + defines = [], + classificationType = ClassificationTypeNames.Comment) + + [] + member this.Comment_MultiLine_WithLineBreakAfterQuotExp() = + this.VerifyColorizerAtEndOfMarker( + fileContents = """ + let mutliLine x = 5(* Test1MultiLine + Test2MultiLine <@@asdf@@> + Test3MultiLine*) + 1(*Test4*) + """, + marker = "Test3", + defines = [], + classificationType = ClassificationTypeNames.Comment) + + [] + member this.Comment_MultiLine_AfterANumber() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let mutliLine x = 5(* Test1MultiLine + Test2MultiLine <@@asdf@@> + Test3MultiLine*) + 1(*Test4*) + """, + marker = "1(*Test4*)", + defines = [], + classificationType = ClassificationTypeNames.NumericLiteral) + + [] + member this.Comment_Nested_Nested01() = + this.VerifyColorizerAtEndOfMarker( + fileContents = """ + (* L1Nesting + (* L2Nesting + (* L3 Nesting + let l3code = 3 + *) + let l2code = 2 + *) + let l1code = 1 + *) + let l0code = 0 + """, + marker = "let l3", + defines = [], + classificationType = ClassificationTypeNames.Comment) + + [] + member this.Comment_Nested_Nested02() = + this.VerifyColorizerAtEndOfMarker( + fileContents = """ + (* L1Nesting + (* L2Nesting + (* L3 Nesting + let l3code = 3 + *) + let l2code = 2 + *) + let l1code = 1 + *) + let l0code = 0 + """, + marker = "let l2", + defines = [], + classificationType = ClassificationTypeNames.Comment) + + [] + member this.Comment_Nested_Nested03() = + this.VerifyColorizerAtEndOfMarker( + fileContents = """ + (* L1Nesting + (* L2Nesting + (* L3 Nesting + let l3code = 3 + *) + let l2code = 2 + *) + let l1code = 1 + *) + let l0code = 0 + """, + marker = "let l1", + defines = [], + classificationType = ClassificationTypeNames.Comment) + + [] + member this.Comment_Nested_IdentAfterNestedComments() = + this.VerifyColorizerAtEndOfMarker( + fileContents = """ + (* L1Nesting + (* L2Nesting + (* L3 Nesting + let l3code = 3 + *) + let l2code = 2 + *) + let l1code = 1 + *) + let l0code = 0 + """, + marker = "let l0", + defines = [], + classificationType = ClassificationTypeNames.Identifier) + + [] + member this.Comment_CommentInString() = + this.VerifyColorizerAtEndOfMarker( + fileContents = """ + let commentsInString = "...(*test1_comment_in_string_literal*)..." + )""", + marker = "test1", + defines = [], + classificationType = ClassificationTypeNames.StringLiteral) + + [] + member this.Comment_StringInComment() = + this.VerifyColorizerAtEndOfMarker( + fileContents = """ + (* + let commentsInString2 = "...*)test2_stringliteral_in_comment(*..." + *)""", + marker = "test2", + defines = [], + classificationType = ClassificationTypeNames.Comment) + + [] + member this.Comment_Unterminated_KeywordBeforeComment() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + type IPeekPoke = interface(*ML Comment Start + abstract member Peek: unit -> int + abstract member Poke: int -> unit + end + """, + marker = "face(*ML Comment Start", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.Comment_Unterminated_KeywordInComment() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + type IPeekPoke = interface(*ML Comment Start + abstract member Peek: unit -> int + abstract member Poke: int -> unit + end + + type wodget = class + val mutable state: int + interface IPeekPoke with(*Few Lines Later2*) + member x.Poke(n) = x.state <- x.state + n + member x.Peek() = x.state + end + end(*Few Lines Later3*)""", + marker = "with(*Few Lines Later2*)", + defines = [], + classificationType = ClassificationTypeNames.Comment) + + [] + member this.Comment_Unterminated_NestedComments() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + type IPeekPoke = interface(*ML Comment Start + abstract member Peek: unit -> int + abstract member Poke: int -> unit + end + + type wodget = class + val mutable state: int + interface IPeekPoke with(*Few Lines Later2*) + member x.Poke(n) = x.state <- x.state + n + member x.Peek() = x.state + end + member x.HasBeenPoked = (x.state <> 0) + new() = { state = 0 } + end(*Few Lines Later3*)""", + marker = "nd(*Few Lines Later3*)", + defines = [], + classificationType = ClassificationTypeNames.Comment) + + [] + member this.String_AtEnd() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let stringone = "simple string test"(*Simple String*) """, + marker = """est"(*Simple String*)""", + defines = [], + classificationType = ClassificationTypeNames.StringLiteral) + + [] + member this.String_MultiLines() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let stringtwo = "simple test(*MultiLine - First*) + string test"(*MultiLine - Second*)""", + marker = "st(*MultiLine - First*)", + defines = [], + classificationType = ClassificationTypeNames.StringLiteral) + + [] + member this.String_MultiLines_LineBreak() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let stringtwo = "simple test(*MultiLine - First*) + string test"(*MultiLine - Second*) """, + marker = "\"(*MultiLine - Second*) ", + defines = [], + classificationType = ClassificationTypeNames.StringLiteral) + + [] + member this.String_Literal() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let stringthree = @"literal test"(*Literal String*)""", + marker = """st"(*Literal String*)""", + defines = [], + classificationType = ClassificationTypeNames.StringLiteral) + + [] + member this.ByteString_AtEnd() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let bytestringone = "abcdefg"B(*Byte String*)""", + marker = "B(*Byte String*)", + defines = [], + classificationType = ClassificationTypeNames.StringLiteral) + + [] + member this.ByteString_MultiLines() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let bytestringtwo = "simple(*MultiLineB - First*) + string"B(*MultiLineB - Second*)""", + marker = """ing"B(*MultiLineB - Second*)""", + defines = [], + classificationType = ClassificationTypeNames.StringLiteral) + + [] + member this.ByteString_Literal() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let bytestringthree = @"literal"B(*Literal Byte*)""", + marker = """al"B(*Literal Byte*)""", + defines = [], + classificationType = ClassificationTypeNames.StringLiteral) + + [] + member this.EscapedIdentifier_word() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """let ``this is an escaped identifier 123ASDF@#$"`` = 4""", + marker = "this", + defines = [], + classificationType = ClassificationTypeNames.Identifier) + + [] + member this.EscapedIdentifier_SpecialChar() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """let ``this is an escaped identifier 123ASDF@#$"`` = 4""", + marker = "3ASDF@#", + defines = [], + classificationType = ClassificationTypeNames.Identifier) + + [] + member this.EscapedIdentifier_EscapeChar() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """let ``this is an escaped identifier 123ASDF@#$"`` = 4""", + marker = "\"``", + defines = [], + classificationType = ClassificationTypeNames.Identifier) + + /// Regression for 3609 - Colorizer: __SOURCE__ and others colorized as a string + [] + member this.PredefinedIdentifier_SOURCE_DIRECTORY() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let x = __SOURCE_DIRECTORY__(*Test1*)""", + marker = "__(*Test1*)", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.PredefinedIdentifier_SOURCE_FILE() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let y = __SOURCE_FILE__(*Test2*))""", + marker = "__(*Test2*)", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.PredefinedIdentifier_LINE() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let z = __LINE__(*Test3*)""", + marker = "__(*Test3*)", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + // Regression Test for FSB 3566, F# colorizer does not respect numbers + [] + member this.Number_InAnExpression() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """let f x = x + 9""", + marker = "9", + defines = [], + classificationType = ClassificationTypeNames.NumericLiteral) + + // Regression Test for FSB 1778 - Colorization seems to be confused after parsing a comment that contains a verbatim string that contains a \ + [] + member this.Number_AfterCommentWithBackSlash() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """let f (* @"\\" *)x = x + 19(*Marker1*)""", + marker = "9(*Marker1*)", + defines = [], + classificationType = ClassificationTypeNames.NumericLiteral) + + // Regression Test for FSharp1.0:2539 -- lexing @"" strings inside (* *) comments? + [] + member this.Keyword_AfterCommentWithLexingStrings() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + (* + let x = @"\\" + *) + + let(*Marker1*) y = 1 + """, + marker = "t(*Marker1*)", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + // Regression Test for FSB 1380 - Language Service colorizes anything followed by a bang as a keyword + [] + member this.Keyword_LetBang() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let seqExpr = + seq { + let! x = [1 .. 10](*Marker1*) + yield! x(*Marker2*) + do! - = ()(*Marker3*) + }""", + marker = "! x = [1 .. 10](*Marker1*)", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.Keyword_Yield() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let seqExpr = + seq { + let! x = [1 .. 10](*Marker1*) + yield! x(*Marker2*) + do! - = ()(*Marker3*) + }""", + marker = "! x(*Marker2*)", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.Keyword_Do() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let seqExpr = + seq { + let! x = [1 .. 10](*Marker1*) + yield! x(*Marker2*) + do! - = ()(*Marker3*) + }""", + marker = "! - = ()(*Marker3*)", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.Keyword_Invalid_Bang() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let seqExpr = + seq { + foo! = true(*Marker1*) + }""", + marker = "! = true(*Marker1*)", + defines = [], + classificationType = ClassificationTypeNames.Identifier) + + [] + [] + [] + //This test case Verify that the color of const is the keyword color + member this.TypeProvider_StaticParameters_Keyword_const() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + type foo = N1.T< const(*Marker1*) "Hello World",2>""", + marker = "t(*Marker1*)", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + // Regression test for FSB 3696 - Colorization doesn't treat #if/else/endif correctly when embedded in a string literal + [] + member this.PreProcessor_InStringLiteral01() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + #if UNDEFINED + let x = "#elseMarker1" + let y = "#endifMarker2" + #else//Marker3 + let x = "#elseMarker4" + let y = "#endifMarker5" + #endif""", + marker = "eMarker1", + defines = [], + classificationType = ClassificationTypeNames.ExcludedCode) + + [] + member this.PreProcessor_InStringLiteral02() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + #if UNDEFINED + let x = "#elseMarker1" + let y = "#endifMarker2" + #else//Marker3 + let x = "#elseMarker4" + let y = "#endifMarker5" + #endif""", + marker = "fMarker2", + defines = [], + classificationType = ClassificationTypeNames.ExcludedCode) + + [] + member this.PreProcessor_ElseKeyword() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + #if UNDEFINED + let x = "#elseMarker1" + let y = "#endifMarker2" + #else//Marker3 + let x = "#elseMarker4" + let y = "#endifMarker5" + #endif""", + marker = "e//Marker3", + defines = [], + classificationType = ClassificationTypeNames.PreprocessorKeyword) + + [] + member this.PreProcessor_InStringLiteral03() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + #if UNDEFINED + let x = "#elseMarker1" + let y = "#endifMarker2" + #else//Marker3 + let x = "#elseMarker4" + let y = "#endifMarker5" + #endif""", + marker = "eMarker4", + defines = [], + classificationType = ClassificationTypeNames.StringLiteral) + + [] + member this.PreProcessor_InStringLiteral04() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + #if UNDEFINED + let x = "#elseMarker1" + let y = "#endifMarker2" + #else//Marker3 + let x = "#elseMarker4" + let y = "#endifMarker5" + #endif""", + marker = "fMarker5", + defines = [], + classificationType = ClassificationTypeNames.StringLiteral) + + // Regression test for FSHARP1.0:4279 + [] + member this.Keyword_OCaml_asr() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let foo a = + match a with + | Some(asr, b) -> () + |_ -> ()""", + marker = "asr", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.Keyword_OCaml_land() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let foo a = + match a with + | Some(land, b) -> () + |_ -> ()""", + marker = "land", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.Keyword_OCaml_lor() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let foo a = + match a with + | Some(lor, b) -> () + |_ -> ()""", + marker = "lor", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.Keyword_OCaml_lsl() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let foo a = + match a with + | Some(lsl, b) -> () + |_ -> ()""", + marker = "lsl", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.Keyword_OCaml_lsr() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let foo a = + match a with + | Some(lsr, b) -> () + |_ -> ()""", + marker = "lsr", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.Keyword_OCaml_lxor() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let foo a = + match a with + | Some(lxor, b) -> () + |_ -> ()""", + marker = "lxor", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.Keyword_OCaml_mod() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let foo a = + match a with + | Some(mod, b) -> () + |_ -> ()""", + marker = "mod", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + member this.Keyword_OCaml_sig() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + let foo a = + match a with + | Some(sig, b) -> () + |_ -> ()""", + marker = "sig", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + member this.InactiveCode(marker: string, classificationType: string) = + let fileContents = """ + #if UNDEFINED + (*Inactive Code1*)let notLegit1 x = x + #else + #if UNDEFINED + (*Inactive Code2*)let notLegit2 x = x + #else + #if UNDEFINED + (*Inactive Code3*)let notLegit3 x = x + #else + #if UNDEFINED + (*Inactive Code4*)let notLegit4 x = x + #else + #if UNDEFINED + (*Inactive Code5*)let notLegit5 x = x + #else + (*Active Code5*)let legitCode5 x = x + #endif + (*Active Code4*)let legitCode4 x = x + #endif + (*Active Code3*)let legitCode3 x = x + #endif + + (*Active Code2*)let legitCode2 x = x + #endif + (*Active Code1*)let legitCode1 x = x + #endif + + #if DEFINED + (*Active Code6*)let legitCode6 x = x + #if DEFINED + (*Active Code7*)let legitCode7 x = x + #else + (*Inactive Code7*)let notLegit7 x = x + #endif + #else + (*Inactive Code6*)let notLegit6 x = x + #endif + """ + + this.VerifyColorizerAtEndOfMarker(fileContents, marker, ["DEFINED"], classificationType) + + + [] + member public this.Colorizer_AtString() = + this.VerifyColorizerAtEndOfMarker("let s = @\"Bob\"", + marker = "let s = @\"B", + defines = [], + classificationType = ClassificationTypeNames.StringLiteral) + + + [] + [] + [] + member public this.Regression_Bug4860(marker: string, classificationType: string) = + this.VerifyColorizerAtStartOfMarker( + fileContents = " + let x = __SOURCE_DIRECTORY__(*Test1*) + let y = __SOURCE_FILE__(*Test2*) + let z = __LINE__(*Test3*)", + marker = marker, + defines = [], + classificationType = classificationType) + + + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + member public this.Number_Regression_Bug3566(marker: string, classificationType: string) = + this.VerifyColorizerAtEndOfMarker( + fileContents = + "let n = 123 + let l = [12..15] + let l2 = [12 .. 15] + let l3 = [ 12 .. 15 ] + // comment1: 1234 + (* comment2: 1234 *) + let other = 0x4, 0b0100, 4L, 4UL, 4u, 4s, 4us, 4y, 4uy, 4.0, 4.0f, 4N, 4I, 1M, 123", + marker = marker, + defines = [], + classificationType = classificationType) + + + /// FEATURE: Hash commands in .fsx files are colorized in PreprocessorKeyword color + [] + member public this.Preprocessor_InFsxFile_StartOfMarker(marker: string, classificationType: string) = + this.VerifyColorizerAtStartOfMarker( + fileContents = + "#reference @\"\" + #load @\"\" + #I <--hash I + #time @\"\"", + marker = marker, + defines = [], + classificationType = classificationType, + isScriptFile = true) + + + /// FEATURE: Hash commands in .fsx files are colorized in PreprocessorKeyword color + [] + [] + [] + member public this.Preprocessor_InFsxFile_EndOfMarker(marker: string, classificationType: string) = + this.VerifyColorizerAtEndOfMarker( + fileContents = + "#reference @\"\" + #load @\"\" + #I <--hash I + #time @\"\"", + marker = marker, + defines = [], + classificationType = classificationType, + isScriptFile = true) + + + /// FEATURE: Script-specific hash commands do not show up in blue in .fs files. + [] + member public this.Preprocessor_InFsFile_StartOfMarker(marker: string, classificationType: string) = + this.VerifyColorizerAtStartOfMarker( + fileContents = + "#reference @\"\" + #load @\"\" + #I <--hash I + #time @\"\"", + marker = marker, + defines = [], + classificationType = classificationType) + + + /// FEATURE: Script-specific hash commands do not show up in blue in .fs files. + [] + [] + [] + member public this.Preprocessor_InFsFile_EndOfMarker(marker: string, classificationType: string) = + this.VerifyColorizerAtEndOfMarker( + fileContents = + "#reference @\"\" + #load @\"\" + #I <--hash I + #time @\"\"", + marker = marker, + defines = [], + classificationType = classificationType) + + /// FEATURE: Nested (* *) comments are allowed and will be colorized with CommentColor. Only the final *) causes the comment to close. + [] + [] + [] + member public this.Comment_AfterCommentBlock(marker: string, classificationType: string) = + this.VerifyColorizerAtEndOfMarker( + fileContents = + "(*Bob*)type Bob() = class end + (* + (* + (*Alice*)type Alice() = class end + *) + *) + (*Charles*)type Charles() = class end", + marker = marker, + defines = [], + classificationType = classificationType) + + + /// BUG: The comment used to be colored in black. + [] + member public this.Regression_Bug1596() = + this.VerifyColorizerAtEndOfMarker( + fileContents = " let 2d (* Identifiers cannot start with numbers *)", + marker = "let 2d (* Ide", + defines = [], + classificationType = ClassificationTypeNames.Comment) + + + /// FEATURE: Code inside #if\#else\#endif blocks is colored with InactiveCodeColor depending on defines. This works for nested #if blocks as well. + [] + [] + [] + [] + [] + [] + member public this.Preprocessor_AfterPreprocessorBlock(marker: string, classificationType: string) = + this.VerifyColorizerAtEndOfMarker( + fileContents = + "(*Bob*)type Bob() = class end + #if UNDEFINED + #if UNDEFINED + (*Alice*)type Alice() = class end + #else + (*Tom*)type Tom() = class end + #endif + #else + #if UNDEFINED + (*Maurice*)type Maurice() = class end + #else + (*Larry*)type Larry() = class end + #endif + #endif + (*Charles*)type Charles() = class end", + marker = marker, + defines = [], + classificationType = classificationType) + + + // Wrong "#else" in "#if" should be ignored + [] + [] + member public this.Preprocessor_InvalidElseDirectiveIgnored(marker: string, classificationType: string) = + this.VerifyColorizerAtEndOfMarker( + fileContents = + "#if UNDEFINED + (*Alice*)type Alice() = class end + (**) #else + (*Larry*)type Larry() = class end + #endif", + marker = marker, + defines = [], + classificationType = classificationType) + + + /// FEATURE: Code inside #if\#else\#endif blocks is colored with InactiveCodeColor depending on defines. This works for nested #if blocks as well. + [] + [] + [] + [] + [] + [] + member public this.Preprocessor_AfterPreprocessorBlockWithDefines(marker: string, classificationType: string) = + this.VerifyColorizerAtEndOfMarker( + fileContents = + "(*Bob*)type Bob() = class end + #if UNDEFINED + #if UNDEFINED + (*Alice*)type Alice() = class end + #else + (*Tom*)type Tom() = class end + #endif + #else + #if UNDEFINED + (*Maurice*)type Maurice() = class end + #else + (*Larry*)type Larry() = class end + #endif + #endif + (*Charles*)type Charles() = class end", + marker = marker, + defines = [], + classificationType = classificationType) + + + /// FEATURE: Preprocessor keywords #light\#if\#else\#endif are colored with the PreprocessorKeyword color. + /// FEATURE: All code in the inactive side of #if\#else\#endif is colored with with InactiveCode color. + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + [] + member public this.Preprocessor_Keywords(marker: string, classificationType: string) = + this.VerifyColorizerAtStartOfMarker( + fileContents = + "#light (*Light*) + #if UNDEFINED //(*If*) + let x = 1(*Inactive*) + #else //(*Else*) + let(*Active*) x = 1 + #endif //(*Endif*)", + marker = marker, + defines = [], + classificationType = classificationType) + + + /// FEATURE: Preprocessor extended grammar basic check. + /// FEATURE: More extensive grammar test is done in compiler unit tests + [] + member public this.Preprocesso_ExtendedIfGrammar_Basic01() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + #if UNDEFINED || !UNDEFINED // Extended #if + let x = "activeCode" + #else + let x = "inactiveCode" + #endif + """, + marker = "activeCode", + defines = [], + classificationType = ClassificationTypeNames.StringLiteral) + + + [] + member public this.Preprocessor_ExtendedIfGrammar_Basic02() = + this.VerifyColorizerAtStartOfMarker( + fileContents = """ + #if UNDEFINED || !UNDEFINED // Extended #if + let x = "activeCode" + #else + let x = "inactiveCode" + #endif + """, + marker = "inactiveCode", + defines = [], + classificationType = ClassificationTypeNames.ExcludedCode) + + + /// #else / #endif in multiline strings is ignored + [] + member public this.Preprocessor_DirectivesInString() = + this.VerifyColorizerAtStartOfMarker( + fileContents = + "#light + + #if DEFINED + let s = \" + #else + \" + let testme = 1 + #endif", + marker = "let testme", + defines = ["DEFINED"], + classificationType = ClassificationTypeNames.Keyword) + + + /// Bug 2076 - String literals were causing the endif stack information to be discarded + [] + [] + [] + member public this.Preprocessor_KeywordsWithStrings(marker: string, classificationType: string) = + this.VerifyColorizerAtStartOfMarker( + fileContents = + "#light (*Light*) + let x1 = \"string1\" + #if UNDEFINED //(*If*) + let x2 = \"string2\" + #else //(*Else*) + let x3 = \"string3\" + #endif //(*Endif*) + let x4 = \"string4\"", + marker = marker, + defines = [], + classificationType = classificationType) + + + [] + member public this.Comment_VerbatimStringInComment_Bug1778() = + this.VerifyColorizerAtStartOfMarker( + fileContents = + "#light + (* @\"\\\" *) let a = 0", + marker = "le", + defines = [], + classificationType = ClassificationTypeNames.Keyword) + + + [] + member public this.Preprocessor_KeywordsWrongIf_Bug1577() = + this.VerifyColorizerAtStartOfMarker( + fileContents = + "#if !!!!!!!!!!!!!!!COMPILED + #endif", + marker = "!!COMPILED", + defines = [], + classificationType = ClassificationTypeNames.Identifier) + + + // This was an off-by-one bug in the replacement Colorizer + [] + member public this.Keyword_LastCharacterOfKeyword() = + this.VerifyColorizerAtEndOfMarker( + fileContents = "(*Bob*)type Bob() = int", + marker = "(*Bob*)typ", + defines = [], + classificationType = ClassificationTypeNames.Keyword) diff --git a/vsintegration/tests/unittests/VisualFSharp.Unittests.fsproj b/vsintegration/tests/unittests/VisualFSharp.Unittests.fsproj index 38fcc5afe42..d8f00a66a8f 100644 --- a/vsintegration/tests/unittests/VisualFSharp.Unittests.fsproj +++ b/vsintegration/tests/unittests/VisualFSharp.Unittests.fsproj @@ -41,7 +41,6 @@ - @@ -85,6 +84,8 @@ VisualFSharp.Unittests.dll.config + + @@ -116,6 +117,7 @@ + @@ -124,6 +126,23 @@ + + + + $(FSharpSourcesRoot)\..\packages\Microsoft.CodeAnalysis.Common.$(RoslynVersion)\lib\net45\Microsoft.CodeAnalysis.dll + + + $(FSharpSourcesRoot)\..\packages\Microsoft.CodeAnalysis.EditorFeatures.$(RoslynVersion)\lib\net46\Microsoft.CodeAnalysis.EditorFeatures.dll + + + $(FSharpSourcesRoot)\..\packages\Microsoft.CodeAnalysis.EditorFeatures.Text.$(RoslynVersion)\lib\net46\Microsoft.CodeAnalysis.EditorFeatures.Text.dll + + + $(FSharpSourcesRoot)\..\packages\Microsoft.CodeAnalysis.Workspaces.Common.$(RoslynVersion)\lib\net45\Microsoft.CodeAnalysis.Workspaces.dll + + + $(FSharpSourcesRoot)\..\packages\Microsoft.VisualStudio.LanguageServices.$(RoslynVersion)\lib\net46\Microsoft.VisualStudio.LanguageServices.dll + True $(NUnitLibDir)\nunit.framework.dll @@ -147,6 +166,11 @@ {a437a6ec-5323-47c2-8f86-e2cac54ff152} True + + FSharp.Editor + {65e0e82a-eace-4787-8994-888674c2fe87} + True + VisualFSharp.Salsa {fbd4b354-dc6e-4032-8ec7-c81d8dfb1af7} From 2b4b731419bafbd825a2bd7a55bcc5e0d791ed29 Mon Sep 17 00:00:00 2001 From: Omar Tawfik Date: Mon, 29 Feb 2016 18:36:39 -0800 Subject: [PATCH 7/9] Final refactoring --- src/fsharp/lexhelp.fs | 7 ++++--- .../src/FSharp.Editor/FSharpBraceMatchingService.fs | 1 + .../src/FSharp.Editor/FSharpColorizationService.fs | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/fsharp/lexhelp.fs b/src/fsharp/lexhelp.fs index 9a6865cb7c6..8d3e4ea4fc4 100644 --- a/src/fsharp/lexhelp.fs +++ b/src/fsharp/lexhelp.fs @@ -3,6 +3,7 @@ module internal Microsoft.FSharp.Compiler.Lexhelp open System +open System.IO open System.Text open Internal.Utilities open Internal.Utilities.Collections @@ -340,14 +341,14 @@ module Keywords = if String.IsNullOrWhiteSpace(filename) then String.Empty else if filename = stdinMockFilename then - System.IO.Directory.GetCurrentDirectory() + Directory.GetCurrentDirectory() else filename |> FileSystem.GetFullPathShim (* asserts that path is already absolute *) - |> System.IO.Path.GetDirectoryName + |> Path.GetDirectoryName KEYWORD_STRING dirname | "__SOURCE_FILE__" -> - KEYWORD_STRING (System.IO.Path.GetFileName((fileOfFileIndex lexbuf.StartPos.FileIndex))) + KEYWORD_STRING (Path.GetFileName((fileOfFileIndex lexbuf.StartPos.FileIndex))) | "__LINE__" -> KEYWORD_STRING (string lexbuf.StartPos.Line) | _ -> diff --git a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs index 6e059a6f3c4..7536f61fa7f 100644 --- a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs +++ b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs @@ -54,6 +54,7 @@ type internal FSharpBraceMatchingService() = } Async.StartAsTask(computation, TaskCreationOptions.None, cancellationToken) + // Helper function to proxy Roslyn types to tests static member FindMatchingBrace(sourceText: SourceText, fileName: string, position: int, cancellationToken: CancellationToken) : Option = let braceMatchingResult = FSharpBraceMatchingService.GetBraceMatchingResult(sourceText, fileName, position, cancellationToken) if braceMatchingResult.HasValue then diff --git a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs index e4b0f347bf3..a94704d2ed1 100644 --- a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs +++ b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs @@ -72,6 +72,7 @@ type internal FSharpColorizationService() = static member private ColorizationDataCache = ConditionalWeakTable() + // Helper function to proxy Roslyn types to tests static member GetColorizationData(sourceText: SourceText, fileName: string, defines: string list, cancellationToken: CancellationToken) : SourceTextColorizationData = FSharpColorizationService.ColorizationDataCache.GetValue(sourceText, fun key -> FSharpColorizationService.ScanSourceText(key, fileName, defines, cancellationToken)) From 30c92a6c723437669e0cfd789960d829a2e52879 Mon Sep 17 00:00:00 2001 From: Omar Tawfik Date: Tue, 1 Mar 2016 15:57:11 -0800 Subject: [PATCH 8/9] Address PR comments --- src/fsharp/vs/IncrementalBuild.fs | 10 +-- src/fsharp/vs/IncrementalBuild.fsi | 3 +- src/fsharp/vs/ServiceLexing.fs | 12 +-- src/fsharp/vs/ServiceLexing.fsi | 2 +- .../FSharpBraceMatchingService.fs | 77 +++++++++-------- .../FSharpColorizationService.fs | 82 ++++++++++--------- .../Salsa/FSharpLanguageServiceTestable.fs | 2 +- .../Tests.LanguageService.General.fs | 2 +- .../Tests.Roslyn.BraceMatchingService.fs | 8 +- .../Tests.Roslyn.ColorizationService.fs | 6 +- 10 files changed, 102 insertions(+), 102 deletions(-) diff --git a/src/fsharp/vs/IncrementalBuild.fs b/src/fsharp/vs/IncrementalBuild.fs index 229367e9084..e4e7ce807da 100755 --- a/src/fsharp/vs/IncrementalBuild.fs +++ b/src/fsharp/vs/IncrementalBuild.fs @@ -927,8 +927,7 @@ type FSharpErrorSeverity = | Warning | Error -type FSharpErrorInfo(id: string, fileName, s:pos, e:pos, severity: FSharpErrorSeverity, message: string, subcategory: string, errorNum: int) = - member __.Id = id +type FSharpErrorInfo(fileName, s:pos, e:pos, severity: FSharpErrorSeverity, message: string, subcategory: string, errorNum: int) = member __.StartLine = Line.toZ s.Line member __.StartLineAlternate = s.Line member __.EndLine = Line.toZ e.Line @@ -940,18 +939,17 @@ type FSharpErrorInfo(id: string, fileName, s:pos, e:pos, severity: FSharpErrorSe member __.Subcategory = subcategory member __.FileName = fileName member __.ErrorNumber = errorNum - member __.WithStart(newStart) = FSharpErrorInfo(id, fileName, newStart, e, severity, message, subcategory, errorNum) - member __.WithEnd(newEnd) = FSharpErrorInfo(id, fileName, s, newEnd, severity, message, subcategory, errorNum) + member __.WithStart(newStart) = FSharpErrorInfo(fileName, newStart, e, severity, message, subcategory, errorNum) + member __.WithEnd(newEnd) = FSharpErrorInfo(fileName, s, newEnd, severity, message, subcategory, errorNum) override __.ToString()= sprintf "%s (%d,%d)-(%d,%d) %s %s %s" fileName (int s.Line) (s.Column + 1) (int e.Line) (e.Column + 1) subcategory (if severity=FSharpErrorSeverity.Warning then "warning" else "error") message /// Decompose a warning or error into parts: position, severity, message, error number static member (*internal*) CreateFromException(exn,warn,trim:bool,fallbackRange:range) = - let id = "FS" + GetErrorNumber(exn).ToString() let m = match GetRangeOfError exn with Some m -> m | None -> fallbackRange let e = if trim then m.Start else m.End let msg = bufs (fun buf -> OutputPhasedError buf exn false) let errorNum = GetErrorNumber exn - FSharpErrorInfo(id, m.FileName, m.Start, e, (if warn then FSharpErrorSeverity.Warning else FSharpErrorSeverity.Error), msg, exn.Subcategory(), errorNum) + FSharpErrorInfo(m.FileName, m.Start, e, (if warn then FSharpErrorSeverity.Warning else FSharpErrorSeverity.Error), msg, exn.Subcategory(), errorNum) /// Decompose a warning or error into parts: position, severity, message, error number static member internal CreateFromExceptionAndAdjustEof(exn,warn,trim:bool,fallbackRange:range, (linesCount:int, lastLength:int)) = diff --git a/src/fsharp/vs/IncrementalBuild.fsi b/src/fsharp/vs/IncrementalBuild.fsi index 52851dc8d9e..f78637d67bb 100755 --- a/src/fsharp/vs/IncrementalBuild.fsi +++ b/src/fsharp/vs/IncrementalBuild.fsi @@ -19,8 +19,7 @@ type internal FSharpErrorSeverity = | Error [] -type internal FSharpErrorInfo = - member Id: string +type internal FSharpErrorInfo = member FileName: string member StartLineAlternate:int member EndLineAlternate:int diff --git a/src/fsharp/vs/ServiceLexing.fs b/src/fsharp/vs/ServiceLexing.fs index bd438c44bea..b838b31b4ce 100755 --- a/src/fsharp/vs/ServiceLexing.fs +++ b/src/fsharp/vs/ServiceLexing.fs @@ -475,7 +475,7 @@ type SingleLineTokenState = [] type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, maxLength: int option, - filename : string, + filename : Option, lexArgsLightOn : lexargs, lexArgsLightOff : lexargs ) = @@ -484,8 +484,8 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, let mutable singleLineTokenState = SingleLineTokenState.BeforeHash let fsx = match filename with - | null -> false - | _ -> CompileOps.IsScript(filename) + | None -> false + | Some(value) -> CompileOps.IsScript(value) // ---------------------------------------------------------------------------------- // This implements post-processing of #directive tokens - not very elegant, but it works... @@ -553,8 +553,8 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, do match filename with - | null -> lexbuf.EndPos <- Internal.Utilities.Text.Lexing.Position.Empty - | _ -> resetLexbufPos filename lexbuf + | None -> lexbuf.EndPos <- Internal.Utilities.Text.Lexing.Position.Empty + | Some(value) -> resetLexbufPos value lexbuf member x.ScanToken(lexintInitial) : Option * FSharpTokenizerLexState = use unwindBP = PushThreadBuildPhaseUntilUnwind (BuildPhase.Parse) @@ -738,7 +738,7 @@ type FSharpLineTokenizer(lexbuf: UnicodeLexing.Lexbuf, LexerStateEncoding.encodeLexCont colorState ncomments position ifdefStack light [] -type FSharpSourceTokenizer(defineConstants : string list, filename : string) = +type FSharpSourceTokenizer(defineConstants : string list, filename : Option) = let lexResourceManager = new Lexhelp.LexResourceManager() let lexArgsLightOn = mkLexargs(filename,defineConstants,LightSyntaxStatus(true,false),lexResourceManager, ref [],DiscardErrorsLogger) diff --git a/src/fsharp/vs/ServiceLexing.fsi b/src/fsharp/vs/ServiceLexing.fsi index 620ef646f85..ba4cc9e2a97 100755 --- a/src/fsharp/vs/ServiceLexing.fsi +++ b/src/fsharp/vs/ServiceLexing.fsi @@ -206,7 +206,7 @@ type internal FSharpLineTokenizer = /// Tokenizer for a source file. Holds some expensive-to-compute resources at the scope of the file. [] type internal FSharpSourceTokenizer = - new : conditionalDefines:string list * fileName:string -> FSharpSourceTokenizer + new : conditionalDefines:string list * fileName:Option -> FSharpSourceTokenizer member CreateLineTokenizer : lineText:string -> FSharpLineTokenizer member CreateBufferTokenizer : bufferFiller:(char[] * int * int -> int) -> FSharpLineTokenizer diff --git a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs index 7536f61fa7f..c5d6cc7c6ec 100644 --- a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs +++ b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs @@ -28,58 +28,33 @@ open Microsoft.FSharp.Compiler.SourceCodeServices [] type internal FSharpBraceMatchingService() = - static member private SupportedBraceTypes = [ + static let supportedBraceTypes = [ ('(', ')'); ('<', '>'); ('[', ']'); ('{', '}'); ] - static member private IgnoredClassificationTypes = [ + static let ignoredClassificationTypes = [ ClassificationTypeNames.Comment; ClassificationTypeNames.StringLiteral; ClassificationTypeNames.ExcludedCode; ] - - interface IBraceMatcher with - member this.FindBracesAsync(document: Document, position: int, cancellationToken: CancellationToken): Task> = - let computation = async { - let! sourceText = document.GetTextAsync(cancellationToken) |> Async.AwaitTask - return - try - FSharpBraceMatchingService.GetBraceMatchingResult(sourceText, document.Name, position, cancellationToken) - with ex -> - Assert.Exception(ex) - reraise() - } - Async.StartAsTask(computation, TaskCreationOptions.None, cancellationToken) - - // Helper function to proxy Roslyn types to tests - static member FindMatchingBrace(sourceText: SourceText, fileName: string, position: int, cancellationToken: CancellationToken) : Option = - let braceMatchingResult = FSharpBraceMatchingService.GetBraceMatchingResult(sourceText, fileName, position, cancellationToken) - if braceMatchingResult.HasValue then - if braceMatchingResult.Value.LeftSpan.Start = position then - Some(braceMatchingResult.Value.RightSpan.Start) - else if braceMatchingResult.Value.RightSpan.Start = position then - Some(braceMatchingResult.Value.LeftSpan.Start) - else - None - else - None - - static member private GetBraceMatchingResult(sourceText: SourceText, fileName: string, position: int, cancellationToken: CancellationToken) : Nullable = + + static let getBraceMatchingResult(sourceText: SourceText, fileName: Option, position: int, cancellationToken: CancellationToken) : Option = if position < 0 || position >= sourceText.Length then - Nullable() + None else - let classificationData = FSharpColorizationService.GetColorizationData(sourceText, fileName, [], cancellationToken) + let completeTextSpan = TextSpan(0, sourceText.Length) + let classificationData = FSharpColorizationService.GetColorizationData(sourceText, completeTextSpan, fileName, [], cancellationToken) let shouldBeIgnored(characterPosition) = match classificationData.GetClassifiedSpan(characterPosition) with | None -> false - | Some(classifiedSpan) -> FSharpBraceMatchingService.IgnoredClassificationTypes |> Seq.contains classifiedSpan.ClassificationType + | Some(classifiedSpan) -> ignoredClassificationTypes |> Seq.contains classifiedSpan.ClassificationType if shouldBeIgnored(position) then - Nullable() + None else let currentCharacter = sourceText.[position] @@ -94,22 +69,46 @@ type internal FSharpBraceMatchingService() = else if currentCharacter = rightBrace then Some(proceedToStartOfString, beforeStartOfString, rightBrace, leftBrace) else None - match FSharpBraceMatchingService.SupportedBraceTypes |> Seq.tryPick(pickBraceType) with - | None -> Nullable() + match supportedBraceTypes |> Seq.tryPick(pickBraceType) with + | None -> None | Some(proceedFunc, stoppingCondition, matchedBrace, nonMatchedBrace) -> let mutable currentPosition = proceedFunc position - let mutable result = Nullable() + let mutable result = None let mutable braceDepth = 0 - while result.HasValue = false && stoppingCondition(currentPosition) = false do + while result.IsSome = false && stoppingCondition(currentPosition) = false do cancellationToken.ThrowIfCancellationRequested() if shouldBeIgnored(currentPosition) = false then if sourceText.[currentPosition] = matchedBrace then braceDepth <- braceDepth + 1 else if sourceText.[currentPosition] = nonMatchedBrace then if braceDepth = 0 then - result <- Nullable(BraceMatchingResult(TextSpan(min position currentPosition, 1), TextSpan(max position currentPosition, 1))) + result <- Some(BraceMatchingResult(TextSpan(min position currentPosition, 1), TextSpan(max position currentPosition, 1))) else braceDepth <- braceDepth - 1 currentPosition <- proceedFunc currentPosition result + + interface IBraceMatcher with + member this.FindBracesAsync(document: Document, position: int, cancellationToken: CancellationToken): Task> = + let computation() = + let sourceText = document.GetTextAsync(cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult() + try match getBraceMatchingResult(sourceText, Some(document.Name), position, cancellationToken) with + | None -> Nullable() + | Some(braceMatchingResult) -> Nullable(braceMatchingResult) + with ex -> + Assert.Exception(ex) + reraise() + Task.Run(computation, cancellationToken) + + // Helper function to proxy Roslyn types to tests + static member FindMatchingBrace(sourceText: SourceText, fileName: Option, position: int, cancellationToken: CancellationToken) : Option = + match getBraceMatchingResult(sourceText, fileName, position, cancellationToken) with + | None -> None + | Some(braceMatchingResult) -> + if braceMatchingResult.LeftSpan.Start = position then + Some(braceMatchingResult.RightSpan.Start) + else if braceMatchingResult.RightSpan.Start = position then + Some(braceMatchingResult.LeftSpan.Start) + else + None diff --git a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs index a94704d2ed1..b684006646f 100644 --- a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs +++ b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs @@ -48,45 +48,9 @@ type internal SourceTextColorizationData(classificationData: seq [, FSharpCommonConstants.FSharpLanguageName)>] type internal FSharpColorizationService() = - interface IEditorClassificationService with - - member this.AddLexicalClassifications(text: SourceText, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - FSharpColorizationService.ClassifySourceTextAsync(text, String.Empty, textSpan, result, cancellationToken).Wait(cancellationToken) - - member this.AddSyntacticClassificationsAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - let sourceText = document.GetTextAsync(cancellationToken).Result - FSharpColorizationService.ClassifySourceTextAsync(sourceText, document.Name, textSpan, result, cancellationToken) - - member this.AddSemanticClassificationsAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - let sourceText = document.GetTextAsync(cancellationToken).Result - FSharpColorizationService.ClassifySourceTextAsync(sourceText, document.Name, textSpan, result, cancellationToken) - - member this.AdjustStaleClassification(text: SourceText, classifiedSpan: ClassifiedSpan) : ClassifiedSpan = - let result = new List() - FSharpColorizationService.ClassifySourceTextAsync(text, String.Empty, classifiedSpan.TextSpan, result, CancellationToken.None).Wait() - if result.Any() then - result.First() - else - new ClassifiedSpan(ClassificationTypeNames.WhiteSpace, classifiedSpan.TextSpan) - - - static member private ColorizationDataCache = ConditionalWeakTable() - - // Helper function to proxy Roslyn types to tests - static member GetColorizationData(sourceText: SourceText, fileName: string, defines: string list, cancellationToken: CancellationToken) : SourceTextColorizationData = - FSharpColorizationService.ColorizationDataCache.GetValue(sourceText, fun key -> FSharpColorizationService.ScanSourceText(key, fileName, defines, cancellationToken)) - - static member private ClassifySourceTextAsync(sourceText: SourceText, fileName: string, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - Task.Run(fun () -> - try - let classificationData = FSharpColorizationService.GetColorizationData(sourceText, fileName, [], cancellationToken) - result.AddRange(classificationData.Tokens |> Seq.filter(fun token -> textSpan.Start <= token.TextSpan.Start && token.TextSpan.End <= textSpan.End)) - with ex -> - Assert.Exception(ex) - reraise() - ) - - static member private ScanSourceText(sourceText: SourceText, fileName: string, defines: string list, cancellationToken: CancellationToken): SourceTextColorizationData = + static let colorizationDataCache = ConditionalWeakTable<(SourceText * TextSpan * Option), SourceTextColorizationData>() + + static let scanSourceText(sourceText: SourceText, textSpan: TextSpan, fileName: Option, defines: string list, cancellationToken: CancellationToken): SourceTextColorizationData = let mutable runningLexState = ref(0L) let result = new List() let sourceTokenizer = FSharpSourceTokenizer(defines, fileName) @@ -133,9 +97,47 @@ type internal FSharpColorizationService() = result.Add(new ClassifiedSpan(classificationType, textSpan)) startPosition <- endPosition - for i = 0 to sourceText.Lines.Count - 1 do + let scanStartLine = sourceText.Lines.GetLineFromPosition(textSpan.Start).LineNumber + let scanEndLine = sourceText.Lines.GetLineFromPosition(textSpan.End).LineNumber + + for i = scanStartLine to scanEndLine do cancellationToken.ThrowIfCancellationRequested() let currentLine = sourceText.Lines.Item(i) scanSourceLine(currentLine, runningLexState) SourceTextColorizationData(result) + + static let classifySourceTextAsync(sourceText: SourceText, fileName: Option, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = + Task.Run(fun () -> + try + let classificationData = FSharpColorizationService.GetColorizationData(sourceText, textSpan, fileName, [], cancellationToken) + result.AddRange(classificationData.Tokens |> Seq.filter(fun token -> textSpan.Start <= token.TextSpan.Start && token.TextSpan.End <= textSpan.End)) + with ex -> + Assert.Exception(ex) + reraise() + ) + + interface IEditorClassificationService with + + member this.AddLexicalClassifications(text: SourceText, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = + classifySourceTextAsync(text, None, textSpan, result, cancellationToken).Wait(cancellationToken) + + member this.AddSyntacticClassificationsAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = + let sourceText = document.GetTextAsync(cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult() + classifySourceTextAsync(sourceText, Some(document.Name), textSpan, result, cancellationToken) + + member this.AddSemanticClassificationsAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = + let sourceText = document.GetTextAsync(cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult() + classifySourceTextAsync(sourceText, Some(document.Name), textSpan, result, cancellationToken) + + member this.AdjustStaleClassification(text: SourceText, classifiedSpan: ClassifiedSpan) : ClassifiedSpan = + let result = new List() + classifySourceTextAsync(text, None, classifiedSpan.TextSpan, result, CancellationToken.None).Wait() + if result.Any() then + result.First() + else + new ClassifiedSpan(ClassificationTypeNames.WhiteSpace, classifiedSpan.TextSpan) + + // Helper function to proxy Roslyn types to tests + static member GetColorizationData(sourceText: SourceText, textSpan: TextSpan, fileName: Option, defines: string list, cancellationToken: CancellationToken) : SourceTextColorizationData = + colorizationDataCache.GetValue((sourceText, textSpan, fileName), fun key -> scanSourceText(sourceText, textSpan, fileName, defines, cancellationToken)) diff --git a/vsintegration/tests/Salsa/FSharpLanguageServiceTestable.fs b/vsintegration/tests/Salsa/FSharpLanguageServiceTestable.fs index 5d6fe745bac..c8d84873989 100644 --- a/vsintegration/tests/Salsa/FSharpLanguageServiceTestable.fs +++ b/vsintegration/tests/Salsa/FSharpLanguageServiceTestable.fs @@ -208,7 +208,7 @@ type internal FSharpLanguageServiceTestable() as this = let filename = VsTextLines.GetFilename buffer let rdt = this.ServiceProvider.RunningDocumentTable let defines = this.ProjectSitesAndFiles.GetDefinesForFile(rdt, filename) - let sourceTokenizer = FSharpSourceTokenizer(defines,filename) + let sourceTokenizer = FSharpSourceTokenizer(defines,Some(filename)) sourceTokenizer.CreateLineTokenizer(source)) let colorizer = new FSharpColorizer(this.CloseColorizer, buffer, scanner) diff --git a/vsintegration/tests/unittests/Tests.LanguageService.General.fs b/vsintegration/tests/unittests/Tests.LanguageService.General.fs index 527de408d62..a34c535a47b 100644 --- a/vsintegration/tests/unittests/Tests.LanguageService.General.fs +++ b/vsintegration/tests/unittests/Tests.LanguageService.General.fs @@ -304,7 +304,7 @@ EdmxFile let filename = "test.fs" let defines = [ "COMPILED"; "EDITING" ] - FSharpSourceTokenizer(defines,filename).CreateLineTokenizer(source)) + FSharpSourceTokenizer(defines,Some(filename)).CreateLineTokenizer(source)) let cm = Microsoft.VisualStudio.FSharp.LanguageService.TokenColor.Comment let kw = Microsoft.VisualStudio.FSharp.LanguageService.TokenColor.Keyword diff --git a/vsintegration/tests/unittests/Tests.Roslyn.BraceMatchingService.fs b/vsintegration/tests/unittests/Tests.Roslyn.BraceMatchingService.fs index 8a1e7a9f03d..424250f52ac 100644 --- a/vsintegration/tests/unittests/Tests.Roslyn.BraceMatchingService.fs +++ b/vsintegration/tests/unittests/Tests.Roslyn.BraceMatchingService.fs @@ -18,7 +18,7 @@ type BraceMatchingServiceTests() = let markerPosition = fileContents.IndexOf(marker) Assert.IsTrue(markerPosition >= 0, "Cannot find marker '{0}' in file contents", marker) - match FSharpBraceMatchingService.FindMatchingBrace(SourceText.From(fileContents), "test.fs", markerPosition, CancellationToken.None) with + match FSharpBraceMatchingService.FindMatchingBrace(SourceText.From(fileContents), Some("test.fs"), markerPosition, CancellationToken.None) with | None -> () | Some(foundMatch) -> Assert.Fail("Found match for brace at position '{0}'", foundMatch) @@ -28,7 +28,7 @@ type BraceMatchingServiceTests() = let endMarkerPosition = fileContents.IndexOf(endMarker) Assert.IsTrue(endMarkerPosition >= 0, "Cannot find end marker '{0}' in file contents", endMarkerPosition) - match FSharpBraceMatchingService.FindMatchingBrace(SourceText.From(fileContents), "test.fs", startMarkerPosition, CancellationToken.None) with + match FSharpBraceMatchingService.FindMatchingBrace(SourceText.From(fileContents), Some("test.fs"), startMarkerPosition, CancellationToken.None) with | None -> Assert.Fail("Didn't find a match for brace at position '{0}", startMarkerPosition) | Some(foundMatch) -> Assert.AreEqual(endMarkerPosition, foundMatch, "Found match at incorrect position") @@ -110,10 +110,10 @@ type BraceMatchingServiceTests() = [] [] [] - [] + [startsInString")>] member this.BraceStartingOrEndingInStringShouldnotBeMatched(startMarker: string) = let code = " let x = \"stringValue\" + (endsInString + \" )endsInString startsInString" this.VerifyNoBraceMatch(code, startMarker) diff --git a/vsintegration/tests/unittests/Tests.Roslyn.ColorizationService.fs b/vsintegration/tests/unittests/Tests.Roslyn.ColorizationService.fs index cac952d3905..9949ebfeb93 100644 --- a/vsintegration/tests/unittests/Tests.Roslyn.ColorizationService.fs +++ b/vsintegration/tests/unittests/Tests.Roslyn.ColorizationService.fs @@ -14,8 +14,9 @@ open Microsoft.VisualStudio.FSharp.Editor type ColorizationServiceTests() = member private this.VerifyColorizerAtStartOfMarker(fileContents: string, marker: string, defines: string list, classificationType: string, ?isScriptFile: bool) = + let textSpan = TextSpan(0, fileContents.Length) let fileName = if isScriptFile.IsSome && isScriptFile.Value then "test.fsx" else "test.fs" - let colorizationData = FSharpColorizationService.GetColorizationData(SourceText.From(fileContents), fileName, defines, CancellationToken.None) + let colorizationData = FSharpColorizationService.GetColorizationData(SourceText.From(fileContents), textSpan, Some(fileName), defines, CancellationToken.None) let markerPosition = fileContents.IndexOf(marker) Assert.IsTrue(markerPosition >= 0, "Cannot find marker '{0}' in file contents", marker) @@ -24,8 +25,9 @@ type ColorizationServiceTests() = | Some(classifiedSpan) -> Assert.AreEqual(classificationType, classifiedSpan.ClassificationType, "Classification data doesn't match for start of marker") member private this.VerifyColorizerAtEndOfMarker(fileContents : string, marker : string, defines: string list, classificationType: string, ?isScriptFile: bool) = + let textSpan = TextSpan(0, fileContents.Length) let fileName = if isScriptFile.IsSome && isScriptFile.Value then "test.fsx" else "test.fs" - let colorizationData = FSharpColorizationService.GetColorizationData(SourceText.From(fileContents), fileName, defines, CancellationToken.None) + let colorizationData = FSharpColorizationService.GetColorizationData(SourceText.From(fileContents), textSpan, Some(fileName), defines, CancellationToken.None) let markerPosition = fileContents.IndexOf(marker) Assert.IsTrue(markerPosition >= 0, "Cannot find marker '{0}' in file contents", marker) From 1da0ea6dbab1b282045cc68ebdfbb40f2baa3cb3 Mon Sep 17 00:00:00 2001 From: Omar Tawfik Date: Thu, 3 Mar 2016 21:51:27 -0800 Subject: [PATCH 9/9] Fix PR comments, add tokens cache per line --- .../FSharpBraceMatchingService.fs | 42 ++-- .../FSharpColorizationService.fs | 205 ++++++++++-------- .../Tests.Roslyn.BraceMatchingService.fs | 12 +- .../Tests.Roslyn.ColorizationService.fs | 24 +- 4 files changed, 148 insertions(+), 135 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs index c5d6cc7c6ec..9eb172ff10c 100644 --- a/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs +++ b/vsintegration/src/FSharp.Editor/FSharpBraceMatchingService.fs @@ -25,6 +25,8 @@ open Microsoft.VisualStudio.Text.Tagging open Microsoft.FSharp.Compiler.Parser open Microsoft.FSharp.Compiler.SourceCodeServices +// TODO: add defines flags if available from project sites and files + [] type internal FSharpBraceMatchingService() = @@ -41,17 +43,17 @@ type internal FSharpBraceMatchingService() = ClassificationTypeNames.ExcludedCode; ] - static let getBraceMatchingResult(sourceText: SourceText, fileName: Option, position: int, cancellationToken: CancellationToken) : Option = + static let getBraceMatchingResult(sourceText: SourceText, fileName: Option, defines: string list, position: int, cancellationToken: CancellationToken) : Option = if position < 0 || position >= sourceText.Length then None else - let completeTextSpan = TextSpan(0, sourceText.Length) - let classificationData = FSharpColorizationService.GetColorizationData(sourceText, completeTextSpan, fileName, [], cancellationToken) - let shouldBeIgnored(characterPosition) = - match classificationData.GetClassifiedSpan(characterPosition) with - | None -> false - | Some(classifiedSpan) -> ignoredClassificationTypes |> Seq.contains classifiedSpan.ClassificationType + let textSpan = TextSpan(characterPosition, 1) + let classifiedSpans = FSharpColorizationService.GetColorizationData(sourceText, textSpan, fileName, defines, cancellationToken) + if classifiedSpans.Any() then + ignoredClassificationTypes |> Seq.contains (classifiedSpans.First().ClassificationType) + else + false if shouldBeIgnored(position) then None @@ -69,7 +71,7 @@ type internal FSharpBraceMatchingService() = else if currentCharacter = rightBrace then Some(proceedToStartOfString, beforeStartOfString, rightBrace, leftBrace) else None - match supportedBraceTypes |> Seq.tryPick(pickBraceType) with + match supportedBraceTypes |> List.tryPick(pickBraceType) with | None -> None | Some(proceedFunc, stoppingCondition, matchedBrace, nonMatchedBrace) -> let mutable currentPosition = proceedFunc position @@ -91,19 +93,19 @@ type internal FSharpBraceMatchingService() = interface IBraceMatcher with member this.FindBracesAsync(document: Document, position: int, cancellationToken: CancellationToken): Task> = - let computation() = - let sourceText = document.GetTextAsync(cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult() - try match getBraceMatchingResult(sourceText, Some(document.Name), position, cancellationToken) with - | None -> Nullable() - | Some(braceMatchingResult) -> Nullable(braceMatchingResult) - with ex -> - Assert.Exception(ex) - reraise() - Task.Run(computation, cancellationToken) - + document.GetTextAsync(cancellationToken).ContinueWith( + fun (sourceTextTask: Task) -> + try match getBraceMatchingResult(sourceTextTask.Result, Some(document.Name), [], position, cancellationToken) with + | None -> Nullable() + | Some(braceMatchingResult) -> Nullable(braceMatchingResult) + with ex -> + Assert.Exception(ex) + reraise() + , TaskContinuationOptions.OnlyOnRanToCompletion) + // Helper function to proxy Roslyn types to tests - static member FindMatchingBrace(sourceText: SourceText, fileName: Option, position: int, cancellationToken: CancellationToken) : Option = - match getBraceMatchingResult(sourceText, fileName, position, cancellationToken) with + static member FindMatchingBrace(sourceText: SourceText, fileName: Option, defines: string list, position: int, cancellationToken: CancellationToken) : Option = + match getBraceMatchingResult(sourceText, fileName, defines, position, cancellationToken) with | None -> None | Some(braceMatchingResult) -> if braceMatchingResult.LeftSpan.Start = position then diff --git a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs index b684006646f..5300f0d7c88 100644 --- a/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs +++ b/vsintegration/src/FSharp.Editor/FSharpColorizationService.fs @@ -28,116 +28,129 @@ open Microsoft.FSharp.Compiler.SourceCodeServices // TODO: add types colorization if available from intellisense // TODO: add defines flags if available from project sites and files -type internal SourceTextColorizationData(classificationData: seq) = - member this.Tokens = classificationData |> Seq.toArray - member this.GetClassifiedSpan(position: int): Option = - let mutable left = 0 - let mutable right = this.Tokens.Length - 1 - let mutable result = None - while result.IsNone && right >= left do - let middle = (left + right) / 2 - let middleToken = this.Tokens.[middle] - if middleToken.TextSpan.End <= position then - left <- middle + 1 - else if middleToken.TextSpan.Start > position then - right <- middle - 1 - else - result <- Some(middleToken) - result +type private SourceLineData(lexStateAtEndOfLine: FSharpTokenizerLexState, hashCode: int, classifiedSpans: IReadOnlyList) = + member val LexStateAtEndOfLine = lexStateAtEndOfLine + member val HashCode = hashCode + member val ClassifiedSpans = classifiedSpans + +type private SourceTextData(lines: int) = + member val Lines = Array.create> lines None [, FSharpCommonConstants.FSharpLanguageName)>] type internal FSharpColorizationService() = - static let colorizationDataCache = ConditionalWeakTable<(SourceText * TextSpan * Option), SourceTextColorizationData>() + static let DataCache = ConditionalWeakTable() + + static let compilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) : string = + match colorKind with + | FSharpTokenColorKind.Comment -> ClassificationTypeNames.Comment + | FSharpTokenColorKind.Identifier -> ClassificationTypeNames.Identifier + | FSharpTokenColorKind.Keyword -> ClassificationTypeNames.Keyword + | FSharpTokenColorKind.String -> ClassificationTypeNames.StringLiteral + | FSharpTokenColorKind.Text -> ClassificationTypeNames.Text + | FSharpTokenColorKind.UpperIdentifier -> ClassificationTypeNames.Identifier + | FSharpTokenColorKind.Number -> ClassificationTypeNames.NumericLiteral + | FSharpTokenColorKind.InactiveCode -> ClassificationTypeNames.ExcludedCode + | FSharpTokenColorKind.PreprocessorKeyword -> ClassificationTypeNames.PreprocessorKeyword + | FSharpTokenColorKind.Operator -> ClassificationTypeNames.Operator + | FSharpTokenColorKind.TypeName -> ClassificationTypeNames.ClassName + | FSharpTokenColorKind.Default | _ -> ClassificationTypeNames.Text + + static let scanAndColorNextToken(lineTokenizer: FSharpLineTokenizer, colorMap: string[], lexState: Ref) : Option = + let tokenInfoOption, nextLexState = lineTokenizer.ScanToken(lexState.Value) + lexState.Value <- nextLexState + if tokenInfoOption.IsSome then + let classificationType = compilerTokenToRoslynToken(tokenInfoOption.Value.ColorClass) + for i = tokenInfoOption.Value.LeftColumn to tokenInfoOption.Value.RightColumn do + Array.set colorMap i classificationType + tokenInfoOption + + static let scanSourceLine(sourceTokenizer: FSharpSourceTokenizer, textLine: TextLine, lineContents: string, lexState: FSharpTokenizerLexState) : SourceLineData = + let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.Text + let lineTokenizer = sourceTokenizer.CreateLineTokenizer(lineContents) + + let previousLextState = ref(lexState) + let mutable tokenInfoOption = scanAndColorNextToken(lineTokenizer, colorMap, previousLextState) + while tokenInfoOption.IsSome do + tokenInfoOption <- scanAndColorNextToken(lineTokenizer, colorMap, previousLextState) + + let mutable startPosition = 0 + let mutable endPosition = startPosition + let classifiedSpans = new List() + + while startPosition < colorMap.Length do + let classificationType = colorMap.[startPosition] + endPosition <- startPosition + while endPosition < colorMap.Length && classificationType = colorMap.[endPosition] do + endPosition <- endPosition + 1 + let textSpan = new TextSpan(textLine.Start + startPosition, endPosition - startPosition) + classifiedSpans.Add(new ClassifiedSpan(classificationType, textSpan)) + startPosition <- endPosition + + SourceLineData(previousLextState.Value, lineContents.GetHashCode(), classifiedSpans) - static let scanSourceText(sourceText: SourceText, textSpan: TextSpan, fileName: Option, defines: string list, cancellationToken: CancellationToken): SourceTextColorizationData = - let mutable runningLexState = ref(0L) - let result = new List() - let sourceTokenizer = FSharpSourceTokenizer(defines, fileName) - - let compilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) = - match colorKind with - | FSharpTokenColorKind.Comment -> ClassificationTypeNames.Comment - | FSharpTokenColorKind.Identifier -> ClassificationTypeNames.Identifier - | FSharpTokenColorKind.Keyword -> ClassificationTypeNames.Keyword - | FSharpTokenColorKind.String -> ClassificationTypeNames.StringLiteral - | FSharpTokenColorKind.Text -> ClassificationTypeNames.Text - | FSharpTokenColorKind.UpperIdentifier -> ClassificationTypeNames.Identifier - | FSharpTokenColorKind.Number -> ClassificationTypeNames.NumericLiteral - | FSharpTokenColorKind.InactiveCode -> ClassificationTypeNames.ExcludedCode - | FSharpTokenColorKind.PreprocessorKeyword -> ClassificationTypeNames.PreprocessorKeyword - | FSharpTokenColorKind.Operator -> ClassificationTypeNames.Operator - | FSharpTokenColorKind.TypeName -> ClassificationTypeNames.ClassName - | FSharpTokenColorKind.Default | _ -> ClassificationTypeNames.Text - - let scanNextToken(lineTokenizer: FSharpLineTokenizer, colorMap: string[], lexState: Ref) = - let tokenInfoOption, currentLexState = lineTokenizer.ScanToken(lexState.Value) - lexState.Value <- currentLexState - match tokenInfoOption with - | None -> false - | Some(tokenInfo) -> - let classificationType = compilerTokenToRoslynToken(tokenInfo.ColorClass) - for i = tokenInfo.LeftColumn to tokenInfo.RightColumn do - Array.set colorMap i classificationType - true - - let scanSourceLine(textLine: TextLine, lexState: Ref) = - let lineTokenizer = sourceTokenizer.CreateLineTokenizer(textLine.Text.ToString(textLine.Span)) - let colorMap = Array.create textLine.Span.Length ClassificationTypeNames.Text - while scanNextToken(lineTokenizer, colorMap, lexState) do () - - let mutable startPosition = 0 - let mutable endPosition = startPosition - while startPosition < colorMap.Length do - let classificationType = colorMap.[startPosition] - endPosition <- startPosition - while endPosition < colorMap.Length && classificationType = colorMap.[endPosition] do - endPosition <- endPosition + 1 - let textSpan = new TextSpan(textLine.Start + startPosition, endPosition - startPosition) - result.Add(new ClassifiedSpan(classificationType, textSpan)) - startPosition <- endPosition - - let scanStartLine = sourceText.Lines.GetLineFromPosition(textSpan.Start).LineNumber - let scanEndLine = sourceText.Lines.GetLineFromPosition(textSpan.End).LineNumber - - for i = scanStartLine to scanEndLine do - cancellationToken.ThrowIfCancellationRequested() - let currentLine = sourceText.Lines.Item(i) - scanSourceLine(currentLine, runningLexState) - - SourceTextColorizationData(result) - - static let classifySourceTextAsync(sourceText: SourceText, fileName: Option, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - Task.Run(fun () -> - try - let classificationData = FSharpColorizationService.GetColorizationData(sourceText, textSpan, fileName, [], cancellationToken) - result.AddRange(classificationData.Tokens |> Seq.filter(fun token -> textSpan.Start <= token.TextSpan.Start && token.TextSpan.End <= textSpan.End)) - with ex -> - Assert.Exception(ex) - reraise() - ) - interface IEditorClassificationService with member this.AddLexicalClassifications(text: SourceText, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - classifySourceTextAsync(text, None, textSpan, result, cancellationToken).Wait(cancellationToken) + result.AddRange(FSharpColorizationService.GetColorizationData(text, textSpan, None, [], cancellationToken)) member this.AddSyntacticClassificationsAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - let sourceText = document.GetTextAsync(cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult() - classifySourceTextAsync(sourceText, Some(document.Name), textSpan, result, cancellationToken) + document.GetTextAsync(cancellationToken).ContinueWith( + fun (sourceTextTask: Task) -> + result.AddRange(FSharpColorizationService.GetColorizationData(sourceTextTask.Result, textSpan, None, [], cancellationToken)) + , TaskContinuationOptions.OnlyOnRanToCompletion) member this.AddSemanticClassificationsAsync(document: Document, textSpan: TextSpan, result: List, cancellationToken: CancellationToken) = - let sourceText = document.GetTextAsync(cancellationToken).ConfigureAwait(false).GetAwaiter().GetResult() - classifySourceTextAsync(sourceText, Some(document.Name), textSpan, result, cancellationToken) + document.GetTextAsync(cancellationToken).ContinueWith( + fun (sourceTextTask: Task) -> + //TODO: Replace with types data when available from intellisense (behaving as AddSyntacticClassificationsAsync() for now) + result.AddRange(FSharpColorizationService.GetColorizationData(sourceTextTask.Result, textSpan, None, [], cancellationToken)) + , TaskContinuationOptions.OnlyOnRanToCompletion) member this.AdjustStaleClassification(text: SourceText, classifiedSpan: ClassifiedSpan) : ClassifiedSpan = - let result = new List() - classifySourceTextAsync(text, None, classifiedSpan.TextSpan, result, CancellationToken.None).Wait() - if result.Any() then - result.First() + let tokens = FSharpColorizationService.GetColorizationData(text, classifiedSpan.TextSpan, None, [], CancellationToken.None) + if tokens.Any() then + tokens.First() else new ClassifiedSpan(ClassificationTypeNames.WhiteSpace, classifiedSpan.TextSpan) - // Helper function to proxy Roslyn types to tests - static member GetColorizationData(sourceText: SourceText, textSpan: TextSpan, fileName: Option, defines: string list, cancellationToken: CancellationToken) : SourceTextColorizationData = - colorizationDataCache.GetValue((sourceText, textSpan, fileName), fun key -> scanSourceText(sourceText, textSpan, fileName, defines, cancellationToken)) + static member GetColorizationData(sourceText: SourceText, textSpan: TextSpan, fileName: Option, defines: string list, cancellationToken: CancellationToken) : List = + try + let sourceTokenizer = FSharpSourceTokenizer(defines, fileName) + let sourceTextData = DataCache.GetValue(sourceText, fun key -> SourceTextData(key.Lines.Count)) + + let startLine = sourceText.Lines.GetLineFromPosition(textSpan.Start).LineNumber + let endLine = sourceText.Lines.GetLineFromPosition(textSpan.End).LineNumber + + // Get the last cached scanned line + let mutable scanStartLine = startLine + while scanStartLine > 0 && sourceTextData.Lines.[scanStartLine - 1].IsNone do + scanStartLine <- scanStartLine - 1 + + let result = new List() + let mutable lexState = if scanStartLine = 0 then 0L else sourceTextData.Lines.[scanStartLine - 1].Value.LexStateAtEndOfLine + + for i = scanStartLine to sourceText.Lines.Count - 1 do + cancellationToken.ThrowIfCancellationRequested() + + let textLine = sourceText.Lines.[i] + let lineContents = textLine.Text.ToString(textLine.Span) + let lineHashCode = lineContents.GetHashCode() + + let mutable lineData = sourceTextData.Lines.[i] + if lineData.IsNone || lineData.Value.HashCode <> lineHashCode then + lineData <- Some(scanSourceLine(sourceTokenizer, textLine, lineContents, lexState)) + + lexState <- lineData.Value.LexStateAtEndOfLine + sourceTextData.Lines.[i] <- lineData + + if startLine <= i && i<= endLine then + result.AddRange(lineData.Value.ClassifiedSpans |> Seq.filter(fun token -> + textSpan.Contains(token.TextSpan.Start) || + textSpan.Contains(token.TextSpan.End - 1) || + (token.TextSpan.Start <= textSpan.Start && textSpan.End <= token.TextSpan.End))) + + result + with ex -> + Assert.Exception(ex) + reraise() \ No newline at end of file diff --git a/vsintegration/tests/unittests/Tests.Roslyn.BraceMatchingService.fs b/vsintegration/tests/unittests/Tests.Roslyn.BraceMatchingService.fs index 424250f52ac..ae4a698c2a7 100644 --- a/vsintegration/tests/unittests/Tests.Roslyn.BraceMatchingService.fs +++ b/vsintegration/tests/unittests/Tests.Roslyn.BraceMatchingService.fs @@ -18,7 +18,7 @@ type BraceMatchingServiceTests() = let markerPosition = fileContents.IndexOf(marker) Assert.IsTrue(markerPosition >= 0, "Cannot find marker '{0}' in file contents", marker) - match FSharpBraceMatchingService.FindMatchingBrace(SourceText.From(fileContents), Some("test.fs"), markerPosition, CancellationToken.None) with + match FSharpBraceMatchingService.FindMatchingBrace(SourceText.From(fileContents), Some("test.fs"), [], markerPosition, CancellationToken.None) with | None -> () | Some(foundMatch) -> Assert.Fail("Found match for brace at position '{0}'", foundMatch) @@ -28,7 +28,7 @@ type BraceMatchingServiceTests() = let endMarkerPosition = fileContents.IndexOf(endMarker) Assert.IsTrue(endMarkerPosition >= 0, "Cannot find end marker '{0}' in file contents", endMarkerPosition) - match FSharpBraceMatchingService.FindMatchingBrace(SourceText.From(fileContents), Some("test.fs"), startMarkerPosition, CancellationToken.None) with + match FSharpBraceMatchingService.FindMatchingBrace(SourceText.From(fileContents), Some("test.fs"), [], startMarkerPosition, CancellationToken.None) with | None -> Assert.Fail("Didn't find a match for brace at position '{0}", startMarkerPosition) | Some(foundMatch) -> Assert.AreEqual(endMarkerPosition, foundMatch, "Found match at incorrect position") @@ -65,7 +65,7 @@ type BraceMatchingServiceTests() = [] [] - member this.BraceInMultiLineCommentShouldnotBeMatched(startMarker: string) = + member this.BraceInMultiLineCommentShouldNotBeMatched(startMarker: string) = let code = " let x = 3 (* This [start @@ -87,7 +87,7 @@ type BraceMatchingServiceTests() = [] [] [] - member this.BraceStartingOrEndingInCommentShouldnotBeMatched(startMarker: string) = + member this.BraceStartingOrEndingInCommentShouldNotBeMatched(startMarker: string) = let code = " let x = 123 + (endsInComment (* )endsInComment ] [] [] - member this.BraceStartingOrEndingInDisabledCodeShouldnotBeMatched(startMarker: string) = + member this.BraceStartingOrEndingInDisabledCodeShouldNotBeMatched(startMarker: string) = let code = " let x = 123 + (endsInDisabledCode #if UNDEFINED @@ -111,7 +111,7 @@ type BraceMatchingServiceTests() = [] [] [startsInString")>] - member this.BraceStartingOrEndingInStringShouldnotBeMatched(startMarker: string) = + member this.BraceStartingOrEndingInStringShouldNotBeMatched(startMarker: string) = let code = " let x = \"stringValue\" + (endsInString + \" )endsInString ] type ColorizationServiceTests() = - - member private this.VerifyColorizerAtStartOfMarker(fileContents: string, marker: string, defines: string list, classificationType: string, ?isScriptFile: bool) = + + member private this.ExtractMarkerData(fileContents: string, marker: string, defines: string list, isScriptFile: Option) = let textSpan = TextSpan(0, fileContents.Length) let fileName = if isScriptFile.IsSome && isScriptFile.Value then "test.fsx" else "test.fs" - let colorizationData = FSharpColorizationService.GetColorizationData(SourceText.From(fileContents), textSpan, Some(fileName), defines, CancellationToken.None) + let tokens = FSharpColorizationService.GetColorizationData(SourceText.From(fileContents), textSpan, Some(fileName), defines, CancellationToken.None) let markerPosition = fileContents.IndexOf(marker) Assert.IsTrue(markerPosition >= 0, "Cannot find marker '{0}' in file contents", marker) - - match colorizationData.GetClassifiedSpan(markerPosition) with + (tokens, markerPosition) + + member private this.VerifyColorizerAtStartOfMarker(fileContents: string, marker: string, defines: string list, classificationType: string, ?isScriptFile: bool) = + let (tokens, markerPosition) = this.ExtractMarkerData(fileContents, marker, defines, isScriptFile) + match tokens |> Seq.tryFind(fun token -> token.TextSpan.Contains(markerPosition)) with | None -> Assert.Fail("Cannot find colorization data for start of marker") | Some(classifiedSpan) -> Assert.AreEqual(classificationType, classifiedSpan.ClassificationType, "Classification data doesn't match for start of marker") - member private this.VerifyColorizerAtEndOfMarker(fileContents : string, marker : string, defines: string list, classificationType: string, ?isScriptFile: bool) = - let textSpan = TextSpan(0, fileContents.Length) - let fileName = if isScriptFile.IsSome && isScriptFile.Value then "test.fsx" else "test.fs" - let colorizationData = FSharpColorizationService.GetColorizationData(SourceText.From(fileContents), textSpan, Some(fileName), defines, CancellationToken.None) - let markerPosition = fileContents.IndexOf(marker) - Assert.IsTrue(markerPosition >= 0, "Cannot find marker '{0}' in file contents", marker) - - match colorizationData.GetClassifiedSpan(markerPosition + marker.Length - 1) with + member private this.VerifyColorizerAtEndOfMarker(fileContents : string, marker: string, defines: string list, classificationType: string, ?isScriptFile: bool) = + let (tokens, markerPosition) = this.ExtractMarkerData(fileContents, marker, defines, isScriptFile) + match tokens |> Seq.tryFind(fun token -> token.TextSpan.Contains(markerPosition + marker.Length - 1)) with | None -> Assert.Fail("Cannot find colorization data for end of marker") | Some(classifiedSpan) -> Assert.AreEqual(classificationType, classifiedSpan.ClassificationType, "Classification data doesn't match for end of marker")