From 9b7edb1699a31c837aac90357092718c8771e666 Mon Sep 17 00:00:00 2001 From: Vasily Kirichenko Date: Fri, 23 Dec 2016 13:59:39 +0300 Subject: [PATCH 1/4] use Roslyn line numbers only in lexer cache --- .../src/FSharp.Editor/Common/CommonHelpers.fs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs index 4ee9660ffe0..587a8de039e 100644 --- a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs +++ b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs @@ -140,13 +140,13 @@ module internal CommonHelpers = // Go backwards to find the last cached scanned line that is valid let scanStartLine = let mutable i = startLine - while i > 0 && (match sourceTextData.[i-1] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do + while i > 0 && (match sourceTextData.[i] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do i <- i - 1 i // Rescan the lines if necessary and report the information let result = new List() - let mutable lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine - 1].Value.LexStateAtEndOfLine + let mutable lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine].Value.LexStateAtEndOfLine for i = scanStartLine to endLine do cancellationToken.ThrowIfCancellationRequested() @@ -319,7 +319,6 @@ module internal CommonHelpers = try let textLine = sourceText.Lines.GetLineFromPosition(position) let textLinePos = sourceText.Lines.GetLinePosition(position) - let lineNumber = textLinePos.Line + 1 // FCS line number let sourceTokenizer = FSharpSourceTokenizer(defines, Some fileName) let lines = sourceText.Lines // We keep incremental data per-document. When text changes we correlate text line-by-line (by hash codes of lines) @@ -327,12 +326,12 @@ module internal CommonHelpers = // Go backwards to find the last cached scanned line that is valid let scanStartLine = - let mutable i = lineNumber - while i > 0 && (match sourceTextData.[i-1] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do + let mutable i = textLinePos.Line + while i >= 0 && (match sourceTextData.[i] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do i <- i - 1 i - let lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine - 1].Value.LexStateAtEndOfLine + let lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine].Value.LexStateAtEndOfLine let lineContents = textLine.Text.ToString(textLine.Span) let lineData = @@ -340,13 +339,13 @@ module internal CommonHelpers = // 1. the line starts at the same overall position // 2. the hash codes match // 3. the start-of-line lex states are the same - match sourceTextData.[lineNumber] with + match sourceTextData.[textLinePos.Line] with | Some data when data.IsValid(textLine) && data.LexStateAtStartOfLine = lexState -> data | _ -> // Otherwise, we recompute let newData = scanSourceLine(sourceTokenizer, textLine, lineContents, lexState) - sourceTextData.[lineNumber] <- Some newData + sourceTextData.[textLinePos.Line] <- Some newData newData getSymbolFromTokens(fileName, lineData.Tokens, textLinePos, lineContents, lookupKind) From 60910aba9f0ca1f2e2607ce15672c68d33c56be3 Mon Sep 17 00:00:00 2001 From: Vasily Kirichenko Date: Sat, 24 Dec 2016 17:46:49 +0300 Subject: [PATCH 2/4] fix and refactor getSymbolAtPosition --- .../src/FSharp.Editor/Common/CommonHelpers.fs | 134 ++++++++---------- 1 file changed, 59 insertions(+), 75 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs index 587a8de039e..dc32f9da609 100644 --- a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs +++ b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs @@ -73,6 +73,13 @@ module internal CommonHelpers = data.[i] <- None i <- i + 1 + /// Go backwards to find the last cached scanned line that is valid. + member x.GetLastValidCachedLine (startLine: int, sourceLines: TextLineCollection) : int = + let mutable i = startLine + while i > 0 && (match x.[i] with Some data -> not (data.IsValid(sourceLines.[i])) | None -> true) do + i <- i - 1 + i + let private dataCache = ConditionalWeakTable() let internal compilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) : string = @@ -126,54 +133,41 @@ module internal CommonHelpers = SourceLineData(textLine.Start, lexState, previousLexState.Value, lineContents.GetHashCode(), classifiedSpans, List.ofSeq tokens) - let getColorizationData(documentKey: DocumentId, sourceText: SourceText, textSpan: TextSpan, fileName: string option, defines: string list, - cancellationToken: CancellationToken) : List = - try - let sourceTokenizer = FSharpSourceTokenizer(defines, fileName) - let lines = sourceText.Lines - // We keep incremental data per-document. When text changes we correlate text line-by-line (by hash codes of lines) - let sourceTextData = dataCache.GetValue(documentKey, fun key -> SourceTextData(lines.Count)) - - let startLine = lines.GetLineFromPosition(textSpan.Start).LineNumber - let endLine = lines.GetLineFromPosition(textSpan.End).LineNumber + let private getSourceLineDatas(documentKey: DocumentId, sourceText: SourceText, startLine: int, endLine: int, fileName: string option, defines: string list, + cancellationToken: CancellationToken) : ResizeArray = + let sourceTokenizer = FSharpSourceTokenizer(defines, fileName) + let lines = sourceText.Lines + // We keep incremental data per-document. When text changes we correlate text line-by-line (by hash codes of lines) + let sourceTextData = dataCache.GetValue(documentKey, fun key -> SourceTextData(lines.Count)) + let scanStartLine = sourceTextData.GetLastValidCachedLine(startLine, lines) - // Go backwards to find the last cached scanned line that is valid - let scanStartLine = - let mutable i = startLine - while i > 0 && (match sourceTextData.[i] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do - i <- i - 1 - i + // Rescan the lines if necessary and report the information + let result = ResizeArray() + let mutable lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine].Value.LexStateAtEndOfLine + + for i = scanStartLine to endLine do + cancellationToken.ThrowIfCancellationRequested() + let textLine = lines.[i] + let lineContents = textLine.Text.ToString(textLine.Span) + + let lineData = + // We can reuse the old data when + // 1. the line starts at the same overall position + // 2. the hash codes match + // 3. the start-of-line lex states are the same + match sourceTextData.[i] with + | Some data when data.IsValid(textLine) && data.LexStateAtStartOfLine = lexState -> + data + | _ -> + // Otherwise, we recompute + let newData = scanSourceLine(sourceTokenizer, textLine, lineContents, lexState) + sourceTextData.[i] <- Some newData + newData - // Rescan the lines if necessary and report the information - let result = new List() - let mutable lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine].Value.LexStateAtEndOfLine - - for i = scanStartLine to endLine do - cancellationToken.ThrowIfCancellationRequested() - let textLine = lines.[i] - let lineContents = textLine.Text.ToString(textLine.Span) - - let lineData = - // We can reuse the old data when - // 1. the line starts at the same overall position - // 2. the hash codes match - // 3. the start-of-line lex states are the same - match sourceTextData.[i] with - | Some data when data.IsValid(textLine) && data.LexStateAtStartOfLine = lexState -> - data - | _ -> - // Otherwise, we recompute - let newData = scanSourceLine(sourceTokenizer, textLine, lineContents, lexState) - sourceTextData.[i] <- Some newData - newData - - lexState <- lineData.LexStateAtEndOfLine - - if startLine <= i then - result.AddRange(lineData.ClassifiedSpans |> Seq.filter(fun token -> - textSpan.Contains(token.TextSpan.Start) || - textSpan.Contains(token.TextSpan.End - 1) || - (token.TextSpan.Start <= textSpan.Start && textSpan.End <= token.TextSpan.End))) + lexState <- lineData.LexStateAtEndOfLine + + if startLine <= i then + result.Add(lineData) // If necessary, invalidate all subsequent lines after endLine if endLine < lines.Count - 1 then @@ -183,6 +177,22 @@ module internal CommonHelpers = sourceTextData.ClearFrom (endLine+1) | None -> () + result + + let getColorizationData(documentKey: DocumentId, sourceText: SourceText, textSpan: TextSpan, fileName: string option, defines: string list, + cancellationToken: CancellationToken) : List = + try + let lines = sourceText.Lines + let startLine = lines.GetLineFromPosition(textSpan.Start).LineNumber + let endLine = lines.GetLineFromPosition(textSpan.End).LineNumber + + // Rescan the lines if necessary and report the information + let result = new List() + for lineData in getSourceLineDatas(documentKey, sourceText, startLine, endLine, fileName, defines, cancellationToken) do + result.AddRange(lineData.ClassifiedSpans |> Seq.filter(fun token -> + textSpan.Contains(token.TextSpan.Start) || + textSpan.Contains(token.TextSpan.End - 1) || + (token.TextSpan.Start <= textSpan.Start && textSpan.End <= token.TextSpan.End))) result with | :? System.OperationCanceledException -> reraise() @@ -319,36 +329,10 @@ module internal CommonHelpers = try let textLine = sourceText.Lines.GetLineFromPosition(position) let textLinePos = sourceText.Lines.GetLinePosition(position) - let sourceTokenizer = FSharpSourceTokenizer(defines, Some fileName) - let lines = sourceText.Lines - // We keep incremental data per-document. When text changes we correlate text line-by-line (by hash codes of lines) - let sourceTextData = dataCache.GetValue(documentKey, fun key -> SourceTextData(lines.Count)) - - // Go backwards to find the last cached scanned line that is valid - let scanStartLine = - let mutable i = textLinePos.Line - while i >= 0 && (match sourceTextData.[i] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do - i <- i - 1 - i - - let lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine].Value.LexStateAtEndOfLine + let sourceLineDatas = getSourceLineDatas(documentKey, sourceText, textLinePos.Line, textLinePos.Line, Some fileName, defines, CancellationToken.None) + assert(sourceLineDatas.Count = 1) let lineContents = textLine.Text.ToString(textLine.Span) - - let lineData = - // We can reuse the old data when - // 1. the line starts at the same overall position - // 2. the hash codes match - // 3. the start-of-line lex states are the same - match sourceTextData.[textLinePos.Line] with - | Some data when data.IsValid(textLine) && data.LexStateAtStartOfLine = lexState -> - data - | _ -> - // Otherwise, we recompute - let newData = scanSourceLine(sourceTokenizer, textLine, lineContents, lexState) - sourceTextData.[textLinePos.Line] <- Some newData - newData - - getSymbolFromTokens(fileName, lineData.Tokens, textLinePos, lineContents, lookupKind) + getSymbolFromTokens(fileName, sourceLineDatas.[0].Tokens, textLinePos, lineContents, lookupKind) with | :? System.OperationCanceledException -> reraise() | ex -> From 38054796f929736d1b081c8d9c87c8917f55023a Mon Sep 17 00:00:00 2001 From: Vasily Kirichenko Date: Fri, 30 Dec 2016 12:17:05 +0300 Subject: [PATCH 3/4] fix after merge --- .../src/FSharp.Editor/Common/CommonHelpers.fs | 29 ++----------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs index f9c2ede222b..5892f1834e1 100644 --- a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs +++ b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs @@ -367,6 +367,7 @@ module internal CommonHelpers = let getSymbolAtPosition(documentKey: DocumentId, sourceText: SourceText, position: int, fileName: string, defines: string list, lookupKind: SymbolLookupKind) : LexerSymbol option = try let lineData, textLinePos, lineContents = getCachedSourceLineData(documentKey, sourceText, position, fileName, defines) + getSymbolFromTokens(fileName, lineData.Tokens, textLinePos, lineContents, lookupKind) with | :? System.OperationCanceledException -> reraise() | ex -> @@ -506,30 +507,4 @@ module internal Extensions = | Some declRange -> declRange.FileName = this.RangeAlternate.FileName | _ -> false - isPrivate && declaredInTheFile - - let glyphMajorToRoslynGlyph = function - | GlyphMajor.Class -> Glyph.ClassPublic - | GlyphMajor.Constant -> Glyph.ConstantPublic - | GlyphMajor.Delegate -> Glyph.DelegatePublic - | GlyphMajor.Enum -> Glyph.EnumPublic - | GlyphMajor.EnumMember -> Glyph.FieldPublic - | GlyphMajor.Event -> Glyph.EventPublic - | GlyphMajor.Exception -> Glyph.ClassPublic - | GlyphMajor.FieldBlue -> Glyph.FieldPublic - | GlyphMajor.Interface -> Glyph.InterfacePublic - | GlyphMajor.Method -> Glyph.MethodPublic - | GlyphMajor.Method2 -> Glyph.MethodPublic - | GlyphMajor.Module -> Glyph.ModulePublic - | GlyphMajor.NameSpace -> Glyph.Namespace - | GlyphMajor.Property -> Glyph.PropertyPublic - | GlyphMajor.Struct -> Glyph.StructurePublic - | GlyphMajor.Typedef -> Glyph.ClassPublic - | GlyphMajor.Type -> Glyph.ClassPublic - | GlyphMajor.Union -> Glyph.EnumPublic - | GlyphMajor.Variable -> Glyph.FieldPublic - | GlyphMajor.ValueType -> Glyph.StructurePublic - | GlyphMajor.Error -> Glyph.Error - | _ -> Glyph.None - - \ No newline at end of file + isPrivate && declaredInTheFile \ No newline at end of file From b76399fcb200383081b89416d972e9f5c45992cd Mon Sep 17 00:00:00 2001 From: Vasily Kirichenko Date: Fri, 30 Dec 2016 12:28:00 +0300 Subject: [PATCH 4/4] fix out of range exn --- .../src/FSharp.Editor/Common/CommonHelpers.fs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs index 5892f1834e1..c9569ebd1a3 100644 --- a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs +++ b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs @@ -328,18 +328,14 @@ module internal CommonHelpers = let private getCachedSourceLineData(documentKey: DocumentId, sourceText: SourceText, position: int, fileName: string, defines: string list) = let textLine = sourceText.Lines.GetLineFromPosition(position) let textLinePos = sourceText.Lines.GetLinePosition(position) - let lineNumber = textLinePos.Line + 1 // FCS line number + let lineNumber = textLinePos.Line let sourceTokenizer = FSharpSourceTokenizer(defines, Some fileName) let lines = sourceText.Lines // We keep incremental data per-document. When text changes we correlate text line-by-line (by hash codes of lines) let sourceTextData = dataCache.GetValue(documentKey, fun key -> SourceTextData(lines.Count)) // Go backwards to find the last cached scanned line that is valid - let scanStartLine = - let mutable i = lineNumber - while i > 0 && (match sourceTextData.[i-1] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do - i <- i - 1 - i - let lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine - 1].Value.LexStateAtEndOfLine + let scanStartLine = sourceTextData.GetLastValidCachedLine(lineNumber, lines) + let lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine].Value.LexStateAtEndOfLine let lineContents = textLine.Text.ToString(textLine.Span) // We can reuse the old data when