From 5290bc3e3af9d9f9c412d787818b773d22dd5338 Mon Sep 17 00:00:00 2001 From: Vasily Kirichenko Date: Wed, 4 Jan 2017 11:30:49 +0300 Subject: [PATCH 1/4] revert "Use Roslyn line numbers only in lexer cache" #2090 --- .../src/FSharp.Editor/Common/CommonHelpers.fs | 139 ++++++++---------- 1 file changed, 65 insertions(+), 74 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs index 3edfcd6768c..a33616027c8 100644 --- a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs +++ b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs @@ -73,13 +73,6 @@ module internal CommonHelpers = data.[i] <- None i <- i + 1 - /// Go backwards to find the last cached scanned line that is valid. - member x.GetLastValidCachedLine (startLine: int, sourceLines: TextLineCollection) : int = - let mutable i = startLine - while i > 0 && (match x.[i] with Some data -> not (data.IsValid(sourceLines.[i])) | None -> true) do - i <- i - 1 - i - let private dataCache = ConditionalWeakTable() let internal compilerTokenToRoslynToken(colorKind: FSharpTokenColorKind) : string = @@ -133,72 +126,66 @@ module internal CommonHelpers = SourceLineData(textLine.Start, lexState, previousLexState.Value, lineContents.GetHashCode(), classifiedSpans, List.ofSeq tokens) - let private getSourceLineDatas(documentKey: DocumentId, sourceText: SourceText, startLine: int, endLine: int, fileName: string option, defines: string list, - cancellationToken: CancellationToken) : ResizeArray = - let sourceTokenizer = FSharpSourceTokenizer(defines, fileName) - let lines = sourceText.Lines - // We keep incremental data per-document. When text changes we correlate text line-by-line (by hash codes of lines) - let sourceTextData = dataCache.GetValue(documentKey, fun key -> SourceTextData(lines.Count)) - let scanStartLine = sourceTextData.GetLastValidCachedLine(startLine, lines) - - // Rescan the lines if necessary and report the information - let result = ResizeArray() - let mutable lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine].Value.LexStateAtEndOfLine - - for i = scanStartLine to endLine do - cancellationToken.ThrowIfCancellationRequested() - let textLine = lines.[i] - let lineContents = textLine.Text.ToString(textLine.Span) - - let lineData = - // We can reuse the old data when - // 1. the line starts at the same overall position - // 2. the hash codes match - // 3. the start-of-line lex states are the same - match sourceTextData.[i] with - | Some data when data.IsValid(textLine) && data.LexStateAtStartOfLine = lexState -> - data - | _ -> - // Otherwise, we recompute - let newData = scanSourceLine(sourceTokenizer, textLine, lineContents, lexState) - sourceTextData.[i] <- Some newData - newData - - lexState <- lineData.LexStateAtEndOfLine - - if startLine <= i then - result.Add(lineData) - - // If necessary, invalidate all subsequent lines after endLine - if endLine < lines.Count - 1 then - match sourceTextData.[endLine+1] with - | Some data -> - if data.LexStateAtStartOfLine <> lexState then - sourceTextData.ClearFrom (endLine+1) - | None -> () - - result - let getColorizationData(documentKey: DocumentId, sourceText: SourceText, textSpan: TextSpan, fileName: string option, defines: string list, cancellationToken: CancellationToken) : List = - try - let lines = sourceText.Lines - let startLine = lines.GetLineFromPosition(textSpan.Start).LineNumber - let endLine = lines.GetLineFromPosition(textSpan.End).LineNumber - - // Rescan the lines if necessary and report the information - let result = new List() - for lineData in getSourceLineDatas(documentKey, sourceText, startLine, endLine, fileName, defines, cancellationToken) do - result.AddRange(lineData.ClassifiedSpans |> Seq.filter(fun token -> - textSpan.Contains(token.TextSpan.Start) || - textSpan.Contains(token.TextSpan.End - 1) || - (token.TextSpan.Start <= textSpan.Start && textSpan.End <= token.TextSpan.End))) - result - with - | :? System.OperationCanceledException -> reraise() - | ex -> - Assert.Exception(ex) - List() + try + let sourceTokenizer = FSharpSourceTokenizer(defines, fileName) + let lines = sourceText.Lines + // We keep incremental data per-document. When text changes we correlate text line-by-line (by hash codes of lines) + let sourceTextData = dataCache.GetValue(documentKey, fun key -> SourceTextData(lines.Count)) + + let startLine = lines.GetLineFromPosition(textSpan.Start).LineNumber + let endLine = lines.GetLineFromPosition(textSpan.End).LineNumber + // Go backwards to find the last cached scanned line that is valid + let scanStartLine = + let mutable i = startLine + while i > 0 && (match sourceTextData.[i-1] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do + i <- i - 1 + i + // Rescan the lines if necessary and report the information + let result = new List() + let mutable lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine - 1].Value.LexStateAtEndOfLine + + for i = scanStartLine to endLine do + cancellationToken.ThrowIfCancellationRequested() + let textLine = lines.[i] + let lineContents = textLine.Text.ToString(textLine.Span) + + let lineData = + // We can reuse the old data when + // 1. the line starts at the same overall position + // 2. the hash codes match + // 3. the start-of-line lex states are the same + match sourceTextData.[i] with + | Some data when data.IsValid(textLine) && data.LexStateAtStartOfLine = lexState -> + data + | _ -> + // Otherwise, we recompute + let newData = scanSourceLine(sourceTokenizer, textLine, lineContents, lexState) + sourceTextData.[i] <- Some newData + newData + + lexState <- lineData.LexStateAtEndOfLine + + if startLine <= i then + result.AddRange(lineData.ClassifiedSpans |> Seq.filter(fun token -> + textSpan.Contains(token.TextSpan.Start) || + textSpan.Contains(token.TextSpan.End - 1) || + (token.TextSpan.Start <= textSpan.Start && textSpan.End <= token.TextSpan.End))) + + // If necessary, invalidate all subsequent lines after endLine + if endLine < lines.Count - 1 then + match sourceTextData.[endLine+1] with + | Some data -> + if data.LexStateAtStartOfLine <> lexState then + sourceTextData.ClearFrom (endLine+1) + | None -> () + result + with + | :? System.OperationCanceledException -> reraise() + | ex -> + Assert.Exception(ex) + List() type private DraftToken = { Kind: LexerSymbolKind @@ -328,14 +315,18 @@ module internal CommonHelpers = let private getCachedSourceLineData(documentKey: DocumentId, sourceText: SourceText, position: int, fileName: string, defines: string list) = let textLine = sourceText.Lines.GetLineFromPosition(position) let textLinePos = sourceText.Lines.GetLinePosition(position) - let lineNumber = textLinePos.Line + let lineNumber = textLinePos.Line + 1 // FCS line number let sourceTokenizer = FSharpSourceTokenizer(defines, Some fileName) let lines = sourceText.Lines // We keep incremental data per-document. When text changes we correlate text line-by-line (by hash codes of lines) let sourceTextData = dataCache.GetValue(documentKey, fun key -> SourceTextData(lines.Count)) // Go backwards to find the last cached scanned line that is valid - let scanStartLine = sourceTextData.GetLastValidCachedLine(lineNumber, lines) - let lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine].Value.LexStateAtEndOfLine + let scanStartLine = + let mutable i = lineNumber + while i > 0 && (match sourceTextData.[i-1] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do + i <- i - 1 + i + let lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine - 1].Value.LexStateAtEndOfLine let lineContents = textLine.Text.ToString(textLine.Span) // We can reuse the old data when From 67b1930ba6dc2e34f3c1a62e35b1e5ecd718f88b Mon Sep 17 00:00:00 2001 From: Vasily Kirichenko Date: Wed, 4 Jan 2017 11:40:58 +0300 Subject: [PATCH 2/4] fix OutOfRangeException in getCachedSourceLineData --- vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs index a33616027c8..7ff137fa7e7 100644 --- a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs +++ b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs @@ -322,7 +322,7 @@ module internal CommonHelpers = let sourceTextData = dataCache.GetValue(documentKey, fun key -> SourceTextData(lines.Count)) // Go backwards to find the last cached scanned line that is valid let scanStartLine = - let mutable i = lineNumber + let mutable i = lineNumber - 1 while i > 0 && (match sourceTextData.[i-1] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do i <- i - 1 i From b26f0d6c15287c7b76ad79102b2594816623466c Mon Sep 17 00:00:00 2001 From: Vasily Kirichenko Date: Wed, 4 Jan 2017 12:41:30 +0300 Subject: [PATCH 3/4] fixed: lexer cache does not work at all --- .../src/FSharp.Editor/Common/CommonHelpers.fs | 19 +++++++++++++------ .../src/FSharp.Editor/FSharp.Editor.fsproj | 2 +- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs index 7ff137fa7e7..cc619eb3340 100644 --- a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs +++ b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs @@ -2,6 +2,8 @@ namespace Microsoft.VisualStudio.FSharp.Editor +#nowarn "1182" + open System open System.Collections.Generic open System.Threading @@ -139,7 +141,7 @@ module internal CommonHelpers = // Go backwards to find the last cached scanned line that is valid let scanStartLine = let mutable i = startLine - while i > 0 && (match sourceTextData.[i-1] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do + while i > 0 && (match sourceTextData.[i] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do i <- i - 1 i // Rescan the lines if necessary and report the information @@ -173,13 +175,13 @@ module internal CommonHelpers = textSpan.Contains(token.TextSpan.End - 1) || (token.TextSpan.Start <= textSpan.Start && textSpan.End <= token.TextSpan.End))) - // If necessary, invalidate all subsequent lines after endLine + // If necessary, invalidate all subsequent lines after endLine if endLine < lines.Count - 1 then match sourceTextData.[endLine+1] with | Some data -> if data.LexStateAtStartOfLine <> lexState then sourceTextData.ClearFrom (endLine+1) - | None -> () + | None -> () result with | :? System.OperationCanceledException -> reraise() @@ -322,9 +324,13 @@ module internal CommonHelpers = let sourceTextData = dataCache.GetValue(documentKey, fun key -> SourceTextData(lines.Count)) // Go backwards to find the last cached scanned line that is valid let scanStartLine = - let mutable i = lineNumber - 1 - while i > 0 && (match sourceTextData.[i-1] with Some data -> not (data.IsValid(lines.[i])) | None -> true) do - i <- i - 1 + let mutable i = lineNumber + while i > 0 && i < lines.Count - 1 && + (match sourceTextData.[i] with + | Some data -> not (data.IsValid(lines.[i])) + | None -> true + ) do + i <- i - 1 i let lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine - 1].Value.LexStateAtEndOfLine let lineContents = textLine.Text.ToString(textLine.Span) @@ -354,6 +360,7 @@ module internal CommonHelpers = let getSymbolAtPosition(documentKey: DocumentId, sourceText: SourceText, position: int, fileName: string, defines: string list, lookupKind: SymbolLookupKind) : LexerSymbol option = try let lineData, textLinePos, lineContents = getCachedSourceLineData(documentKey, sourceText, position, fileName, defines) + let sourceTokenizer = FSharpSourceTokenizer(defines, Some fileName) getSymbolFromTokens(fileName, lineData.Tokens, textLinePos, lineContents, lookupKind) with | :? System.OperationCanceledException -> reraise() diff --git a/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj b/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj index 213a98d79cc..16337c02644 100644 --- a/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj +++ b/vsintegration/src/FSharp.Editor/FSharp.Editor.fsproj @@ -34,8 +34,8 @@ - + From 468b0c761d2b25095655c30a9507cab687fe912f Mon Sep 17 00:00:00 2001 From: Vasily Kirichenko Date: Thu, 5 Jan 2017 22:57:03 +0300 Subject: [PATCH 4/4] fix finding start line for relexing in getCachedSourceLineData --- .../src/FSharp.Editor/Common/CommonHelpers.fs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs index cc619eb3340..78f04feb792 100644 --- a/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs +++ b/vsintegration/src/FSharp.Editor/Common/CommonHelpers.fs @@ -324,14 +324,14 @@ module internal CommonHelpers = let sourceTextData = dataCache.GetValue(documentKey, fun key -> SourceTextData(lines.Count)) // Go backwards to find the last cached scanned line that is valid let scanStartLine = - let mutable i = lineNumber - while i > 0 && i < lines.Count - 1 && - (match sourceTextData.[i] with - | Some data -> not (data.IsValid(lines.[i])) - | None -> true - ) do - i <- i - 1 - i + let mutable i = min (lines.Count - 1) lineNumber + while i > 0 && + (match sourceTextData.[i] with + | Some data -> not (data.IsValid(lines.[i])) + | None -> true + ) do + i <- i - 1 + i let lexState = if scanStartLine = 0 then 0L else sourceTextData.[scanStartLine - 1].Value.LexStateAtEndOfLine let lineContents = textLine.Text.ToString(textLine.Span)