From 7d38158151372d83e131b155b0816538a8e2dadc Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Tue, 28 Feb 2023 12:20:20 -0600 Subject: [PATCH 01/10] Add Injected languages --- Package.swift | 2 +- .../Extensions/NSRange+/NSRange+TSRange.swift | 15 ++ .../STTextViewController+TextFormation.swift | 2 +- .../Highlighting/Highlighter.swift | 2 +- .../TreeSitter/TreeSitterClient.swift | 239 ++++++++++++++---- 5 files changed, 211 insertions(+), 49 deletions(-) create mode 100644 Sources/CodeEditTextView/Extensions/NSRange+/NSRange+TSRange.swift diff --git a/Package.swift b/Package.swift index 5c1461071..68ec5e51f 100644 --- a/Package.swift +++ b/Package.swift @@ -19,7 +19,7 @@ let package = Package( ), .package( url: "https://github.com/CodeEditApp/CodeEditLanguages.git", - exact: "0.1.11" + from: "0.1.10" ), .package( url: "https://github.com/lukepistrol/SwiftLintPlugin", diff --git a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+TSRange.swift b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+TSRange.swift new file mode 100644 index 000000000..1b3f8a749 --- /dev/null +++ b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+TSRange.swift @@ -0,0 +1,15 @@ +// +// NSRange+TSRange.swift +// +// +// Created by Khan Winter on 2/26/23. +// + +import Foundation +import SwiftTreeSitter + +extension NSRange { + var tsRange: TSRange { + return TSRange(points: .zero..<(.zero), bytes: (UInt32(self.lowerBound) * 2)..<(UInt32(self.upperBound) * 2)) + } +} diff --git a/Sources/CodeEditTextView/Filters/STTextViewController+TextFormation.swift b/Sources/CodeEditTextView/Filters/STTextViewController+TextFormation.swift index 55bba4a2e..0e0ad7cac 100644 --- a/Sources/CodeEditTextView/Filters/STTextViewController+TextFormation.swift +++ b/Sources/CodeEditTextView/Filters/STTextViewController+TextFormation.swift @@ -71,7 +71,7 @@ extension STTextViewController { /// - whitespaceProvider: The whitespace providers to use. /// - indentationUnit: The unit of indentation to use. private func setUpNewlineTabFilters(whitespaceProvider: WhitespaceProviders, indentationUnit: String) { - let newlineFilter: Filter = NewlineFilter(whitespaceProviders: whitespaceProvider) + let newlineFilter: Filter = NewlineProcessingFilter(whitespaceProviders: whitespaceProvider) let tabReplacementFilter: Filter = TabReplacementFilter(indentationUnit: indentationUnit) textFilters.append(contentsOf: [newlineFilter, tabReplacementFilter]) diff --git a/Sources/CodeEditTextView/Highlighting/Highlighter.swift b/Sources/CodeEditTextView/Highlighting/Highlighter.swift index fe4990bca..83016e130 100644 --- a/Sources/CodeEditTextView/Highlighting/Highlighter.swift +++ b/Sources/CodeEditTextView/Highlighting/Highlighter.swift @@ -282,7 +282,7 @@ extension Highlighter: NSTextStorageDelegate { delta: delta) { [weak self] invalidatedIndexSet in let indexSet = invalidatedIndexSet .union(IndexSet(integersIn: editedRange)) - // Only invalidate indices that aren't visible. + // Only invalidate indices that are visible. .intersection(self?.visibleSet ?? .init()) for range in indexSet.rangeView { diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index eea44f160..2b1dd15b6 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -22,9 +22,25 @@ final class TreeSitterClient: HighlightProviding { "CodeEdit.TreeSitterClient" } - internal var parser: Parser - internal var tree: Tree? - internal var languageQuery: Query? + private var primaryLanguage: TreeSitterLanguage + private var languages: [TreeSitterLanguage: Language] = [:] + + class Language { + init(id: TreeSitterLanguage, + parser: Parser, + tree: Tree? = nil, + languageQuery: Query? = nil) { + self.id = id + self.parser = parser + self.tree = tree + self.languageQuery = languageQuery + } + + var id: TreeSitterLanguage + var parser: Parser + var tree: Tree? + var languageQuery: Query? + } private var textProvider: ResolvingQueryCursor.TextProvider @@ -40,24 +56,29 @@ final class TreeSitterClient: HighlightProviding { /// - codeLanguage: The language to set up the parser with. /// - textProvider: The text provider callback to read any text. public init(codeLanguage: CodeLanguage, textProvider: @escaping ResolvingQueryCursor.TextProvider) throws { - parser = Parser() - languageQuery = TreeSitterModel.shared.query(for: codeLanguage.id) - tree = nil + primaryLanguage = codeLanguage.id + languages[codeLanguage.id] = Language(id: codeLanguage.id, + parser: Parser(), + tree: nil, + languageQuery: TreeSitterModel.shared.query(for: codeLanguage.id)) self.textProvider = textProvider if let treeSitterLanguage = codeLanguage.language { - try parser.setLanguage(treeSitterLanguage) + try languages[codeLanguage.id]?.parser.setLanguage(treeSitterLanguage) } } func setLanguage(codeLanguage: CodeLanguage) { - if let treeSitterLanguage = codeLanguage.language { - try? parser.setLanguage(treeSitterLanguage) + // Remove all trees and languages, everything needs to be re-parsed. + for key in languages.keys where key != codeLanguage.id { + languages.removeValue(forKey: key) } + primaryLanguage = codeLanguage.id - // Get rid of the current tree, it needs to be re-parsed. - tree = nil + if let treeSitterLanguage = codeLanguage.language { + try? languages[codeLanguage.id]?.parser.setLanguage(treeSitterLanguage) + } } /// Notifies the highlighter of an edit and in exchange gets a set of indices that need to be re-highlighted. @@ -87,17 +108,12 @@ final class TreeSitterClient: HighlightProviding { return textView.stringForRange(range)?.data(using: String.nativeUTF16Encoding) } - let (oldTree, newTree) = self.calculateNewState(edit: edit, - readBlock: readFunction) - - if oldTree == nil && newTree == nil { - // There was no existing tree, make a new one and return all indexes. - createTree(textView: textView) - completion(IndexSet(integersIn: textView.documentRange.intRange)) - return - } - - let effectedRanges = self.changedByteRanges(oldTree, rhs: newTree).map { $0.range } + let effectedRanges: [NSRange] = findChangedByteRanges( + textView: textView, + edit: edit, + language: languages[primaryLanguage]!, + readBlock: readFunction + ) var rangeSet = IndexSet() effectedRanges.forEach { range in @@ -106,18 +122,52 @@ final class TreeSitterClient: HighlightProviding { completion(rangeSet) } + /// Calculates a series of ranges that have been invalidated by a given edit. + /// - Parameters: + /// - textView: The text view to use for text. + /// - edit: The edit to act on. + /// - language: The language to use. + /// - readBlock: A callback for fetching blocks of text. + /// - Returns: An array of distinct `NSRanges` that need to be re-highlighted. + func findChangedByteRanges(textView: HighlighterTextView, + edit: InputEdit, + language: Language, + readBlock: @escaping Parser.ReadBlock) -> [NSRange] { + let (oldTree, newTree) = calculateNewState(tree: language.tree, + parser: language.parser, + edit: edit, + readBlock: readBlock) + if oldTree == nil && newTree == nil { + // There was no existing tree, make a new one and return all indexes. + languages[language.id]?.tree = createTree(textView: textView, parser: language.parser) + return [NSRange(textView.documentRange.intRange)] + } + + let ranges = changedByteRanges(oldTree, rhs: newTree).map { $0.range } + + languages[language.id]?.tree = newTree + + return ranges + } + + /// Initiates a highlight query. + /// - Parameters: + /// - textView: The text view to use. + /// - range: The range to limit the highlights to. + /// - completion: Called when the query completes. func queryHighlightsFor(textView: HighlighterTextView, range: NSRange, completion: @escaping (([HighlightRange]) -> Void)) { + let language = languages[primaryLanguage]! // Make sure we dont accidentally change the tree while we copy it. self.semaphore.wait() - guard let tree = self.tree?.copy() else { + guard let tree = language.tree?.copy() else { // In this case, we don't have a tree to work with already, so we need to make it and try to // return some highlights - createTree(textView: textView) + language.tree = createTree(textView: textView, parser: language.parser) // This is slightly redundant but we're only doing one check. - guard let treeRetry = self.tree?.copy() else { + guard let treeRetry = language.tree?.copy() else { // Now we can return nothing for real. self.semaphore.signal() completion([]) @@ -125,40 +175,109 @@ final class TreeSitterClient: HighlightProviding { } self.semaphore.signal() - _queryColorsFor(tree: treeRetry, range: range, completion: completion) + completion( + _queryColorsFor(textView: textView, language: language, tree: treeRetry, range: range) + + queryInjectedLanguages(textView: textView, language: language, tree: treeRetry, range: range) + ) return } self.semaphore.signal() - _queryColorsFor(tree: tree, range: range, completion: completion) + completion( + _queryColorsFor(textView: textView, language: language, tree: tree, range: range) + + queryInjectedLanguages(textView: textView, language: language, tree: tree, range: range) + ) } - private func _queryColorsFor(tree: Tree, - range: NSRange, - completion: @escaping (([HighlightRange]) -> Void)) { + private func _queryColorsFor(textView: HighlighterTextView, + language: Language, + tree: Tree, + range: NSRange) -> [HighlightRange] { guard let rootNode = tree.rootNode else { - completion([]) - return + return [] } // This needs to be on the main thread since we're going to use the `textProvider` in // the `highlightsFromCursor` method, which uses the textView's text storage. - guard let cursor = self.languageQuery?.execute(node: rootNode, in: tree) else { - completion([]) - return + guard let cursor = language.languageQuery?.execute(node: rootNode, in: tree) else { + return [] + } + cursor.setRange(range) + + let highlights = highlightsFromCursor(cursor: ResolvingQueryCursor(cursor: cursor)) + + return highlights + } + + private func queryInjectedLanguages(textView: HighlighterTextView, + language: Language, + tree: Tree, + range: NSRange) -> [HighlightRange] { + guard let rootNode = tree.rootNode else { + return [] + } + + guard let cursor = language.languageQuery?.execute(node: rootNode, in: tree) else { + return [] } cursor.setRange(range) - let highlights = self.highlightsFromCursor(cursor: ResolvingQueryCursor(cursor: cursor)) + let languageRanges = self.injectedLanguagesFrom(cursor: cursor) { range, _ in + return textView.stringForRange(range) + } + + var highlights: [HighlightRange] = [] + + for (languageName, ranges) in languageRanges { + guard let language = TreeSitterLanguage(rawValue: languageName) else { + continue + } + + if language == primaryLanguage { + continue + } + + languages[language] = Language(id: language, + parser: Parser(), + tree: nil, + languageQuery: TreeSitterModel.shared.query(for: language)) + + guard let parserLanguage = CodeLanguage + .allLanguages + .first(where: { $0.id == language })? + .language + else { + continue + } + try? languages[language]?.parser.setLanguage(parserLanguage) + languages[language]?.parser.includedRanges = ranges.map { $0.tsRange } + languages[language]?.tree = createTree(textView: textView, parser: languages[language]!.parser) + + for range in ranges { + highlights.append( + contentsOf: _queryColorsFor(textView: textView, + language: languages[language]!, + tree: languages[language]!.tree!, + range: range.range) + ) + } - completion(highlights) + highlights.append( + contentsOf: queryInjectedLanguages(textView: textView, + language: languages[language]!, + tree: languages[language]!.tree!, + range: range) + ) + } + + return highlights } /// Creates a tree. /// - Parameter textView: The text provider to use. - private func createTree(textView: HighlighterTextView) { - self.tree = self.parser.parse(textView.stringForRange(textView.documentRange) ?? "") + private func createTree(textView: HighlighterTextView, parser: Parser) -> Tree? { + return parser.parse(textView.stringForRange(textView.documentRange) ?? "") } /// Resolves a query cursor to the highlight ranges it contains. @@ -169,7 +288,33 @@ final class TreeSitterClient: HighlightProviding { cursor.prepare(with: self.textProvider) return cursor .flatMap { $0.captures } - .map { HighlightRange(range: $0.range, capture: CaptureName.fromString($0.name ?? "")) } + .map { + HighlightRange(range: $0.range, capture: CaptureName.fromString($0.name ?? "")) + } + } + + /// Returns all injected languages from a given cursor. The cursor must be new, + /// having not been used for normal highlight matching. + /// - Parameters: + /// - cursor: The cursor to use for finding injected languages. + /// - textProvider: A callback for efficiently fetching text. + /// - Returns: A map of each language to all the ranges they have been injected into. + private func injectedLanguagesFrom( + cursor: QueryCursor, + textProvider: @escaping ResolvingQueryCursor.TextProvider + ) -> [String: [NamedRange]] { + var languages: [String: [NamedRange]] = [:] + + for match in cursor { + if let injection = match.injection(with: textProvider) { + if languages[injection.name] == nil { + languages[injection.name] = [] + } + languages[injection.name]?.append(injection) + } + } + + return languages } } @@ -180,21 +325,23 @@ extension TreeSitterClient { /// - edit: The edit to apply. /// - readBlock: The block to use to read text. /// - Returns: (The old state, the new state). - private func calculateNewState(edit: InputEdit, + private func calculateNewState(tree: Tree?, + parser: Parser, + edit: InputEdit, readBlock: @escaping Parser.ReadBlock) -> (Tree?, Tree?) { - guard let oldTree = self.tree else { + guard let oldTree = tree else { return (nil, nil) } - self.semaphore.wait() + semaphore.wait() // Apply the edit to the old tree oldTree.edit(edit) - self.tree = self.parser.parse(tree: oldTree, readBlock: readBlock) + let newTree = parser.parse(tree: oldTree, readBlock: readBlock) - self.semaphore.signal() + semaphore.signal() - return (oldTree.copy(), self.tree?.copy()) + return (oldTree.copy(), newTree) } /// Calculates the changed byte ranges between two trees. From 7c28303a9339e11472061f9341fe2355282230a1 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Tue, 28 Feb 2023 14:36:57 -0600 Subject: [PATCH 02/10] Fix lint errors! --- Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index 2b1dd15b6..4b4f797a5 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -226,7 +226,7 @@ final class TreeSitterClient: HighlightProviding { let languageRanges = self.injectedLanguagesFrom(cursor: cursor) { range, _ in return textView.stringForRange(range) } - + var highlights: [HighlightRange] = [] for (languageName, ranges) in languageRanges { From 3921648d55d7e2c6ed61d7642ba0b15a652d45b3 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Tue, 28 Feb 2023 23:44:07 -0600 Subject: [PATCH 03/10] Use read blocks for tree creation --- .../TreeSitter/TreeSitterClient.swift | 83 ++++++++++++------- 1 file changed, 52 insertions(+), 31 deletions(-) diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index 4b4f797a5..a382f33ef 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -96,23 +96,13 @@ final class TreeSitterClient: HighlightProviding { return } - let readFunction: Parser.ReadBlock = { byteOffset, _ in - let limit = textView.documentRange.length - let location = byteOffset / 2 - let end = min(location + (1024), limit) - if location > end { - assertionFailure("location is greater than end") - return nil - } - let range = NSRange(location.. Void)) { + let readBlock = createReadBlock(textView: textView) + let language = languages[primaryLanguage]! // Make sure we dont accidentally change the tree while we copy it. self.semaphore.wait() guard let tree = language.tree?.copy() else { // In this case, we don't have a tree to work with already, so we need to make it and try to // return some highlights - language.tree = createTree(textView: textView, parser: language.parser) + language.tree = createTree(parser: language.parser, readBlock: readBlock) // This is slightly redundant but we're only doing one check. guard let treeRetry = language.tree?.copy() else { @@ -177,7 +169,11 @@ final class TreeSitterClient: HighlightProviding { completion( _queryColorsFor(textView: textView, language: language, tree: treeRetry, range: range) + - queryInjectedLanguages(textView: textView, language: language, tree: treeRetry, range: range) + queryInjectedLanguages(textView: textView, + language: language, + tree: treeRetry, + range: range, + readBlock: readBlock) ) return } @@ -186,7 +182,11 @@ final class TreeSitterClient: HighlightProviding { completion( _queryColorsFor(textView: textView, language: language, tree: tree, range: range) + - queryInjectedLanguages(textView: textView, language: language, tree: tree, range: range) + queryInjectedLanguages(textView: textView, + language: language, + tree: tree, + range: range, + readBlock: readBlock) ) } @@ -213,14 +213,13 @@ final class TreeSitterClient: HighlightProviding { private func queryInjectedLanguages(textView: HighlighterTextView, language: Language, tree: Tree, - range: NSRange) -> [HighlightRange] { - guard let rootNode = tree.rootNode else { + range: NSRange, + readBlock: @escaping Parser.ReadBlock) -> [HighlightRange] { + guard let rootNode = tree.rootNode, + let cursor = language.languageQuery?.execute(node: rootNode, in: tree) else { return [] } - guard let cursor = language.languageQuery?.execute(node: rootNode, in: tree) else { - return [] - } cursor.setRange(range) let languageRanges = self.injectedLanguagesFrom(cursor: cursor) { range, _ in @@ -238,10 +237,12 @@ final class TreeSitterClient: HighlightProviding { continue } - languages[language] = Language(id: language, - parser: Parser(), - tree: nil, - languageQuery: TreeSitterModel.shared.query(for: language)) + if languages[language] == nil { + languages[language] = Language(id: language, + parser: Parser(), + tree: nil, + languageQuery: TreeSitterModel.shared.query(for: language)) + } guard let parserLanguage = CodeLanguage .allLanguages @@ -250,9 +251,11 @@ final class TreeSitterClient: HighlightProviding { else { continue } + try? languages[language]?.parser.setLanguage(parserLanguage) languages[language]?.parser.includedRanges = ranges.map { $0.tsRange } - languages[language]?.tree = createTree(textView: textView, parser: languages[language]!.parser) + languages[language]?.tree = createTree(parser: languages[language]!.parser, + readBlock: readBlock) for range in ranges { highlights.append( @@ -267,17 +270,21 @@ final class TreeSitterClient: HighlightProviding { contentsOf: queryInjectedLanguages(textView: textView, language: languages[language]!, tree: languages[language]!.tree!, - range: range) + range: range, + readBlock: readBlock) ) } return highlights } - /// Creates a tree. - /// - Parameter textView: The text provider to use. - private func createTree(textView: HighlighterTextView, parser: Parser) -> Tree? { - return parser.parse(textView.stringForRange(textView.documentRange) ?? "") + /// Creates a tree-sitter tree. + /// - Parameters: + /// - parser: The parser object to use to parse text. + /// - readBlock: A callback for fetching blocks of text. + /// - Returns: A tree if it could be parsed. + private func createTree(parser: Parser, readBlock: @escaping Parser.ReadBlock) -> Tree? { + return parser.parse(tree: nil, readBlock: readBlock) } /// Resolves a query cursor to the highlight ranges it contains. @@ -319,6 +326,20 @@ final class TreeSitterClient: HighlightProviding { } extension TreeSitterClient { + private func createReadBlock(textView: HighlighterTextView) -> Parser.ReadBlock { + return { byteOffset, _ in + let limit = textView.documentRange.length + let location = byteOffset / 2 + let end = min(location + (1024), limit) + if location > end { + assertionFailure("location is greater than end") + return nil + } + let range = NSRange(location.. Date: Tue, 14 Mar 2023 15:04:13 -0500 Subject: [PATCH 04/10] Broken it again (but it's more efficient) --- Package.swift | 2 +- .../NSRange+/NSRange+InputEdit.swift | 34 ++ .../Extensions/NSRange+/NSRange+TSRange.swift | 3 +- .../Highlighting/HighlightProviding.swift | 3 + .../Highlighting/Highlighter.swift | 2 + .../TreeSitter/TreeSitterClient+Edit.swift | 86 ++++ .../TreeSitterClient+Highlight.swift | 145 +++++++ .../TreeSitterClient+LanguageLayer.swift | 33 ++ .../TreeSitter/TreeSitterClient.swift | 401 ++++++------------ 9 files changed, 434 insertions(+), 275 deletions(-) create mode 100644 Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift create mode 100644 Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift create mode 100644 Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift diff --git a/Package.swift b/Package.swift index 68ec5e51f..216a55eba 100644 --- a/Package.swift +++ b/Package.swift @@ -19,7 +19,7 @@ let package = Package( ), .package( url: "https://github.com/CodeEditApp/CodeEditLanguages.git", - from: "0.1.10" + exact: "0.1.12" ), .package( url: "https://github.com/lukepistrol/SwiftLintPlugin", diff --git a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift index 9994a92df..b2b97f0e2 100644 --- a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift +++ b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift @@ -29,3 +29,37 @@ extension InputEdit { newEndPoint: newEndPoint) } } + +extension NSRange { + // swiftlint:disable line_length + + /// Modifies the range to account for an edit. + /// Largely based on: + /// [tree-sitter](https://github.com/tree-sitter/tree-sitter/blob/ddeaa0c7f534268b35b4f6cb39b52df082754413/lib/src/subtree.c#L691-L720) + mutating func applyInputEdit(_ edit: InputEdit) { + // swiftlint:enable line_length + + let endIndex = NSMaxRange(self) + let isPureInsertion = edit.oldEndByte == edit.startByte + + // Edit is after the range + if (edit.startByte/2) > endIndex { + return + } else if edit.oldEndByte/2 < location { + // If the edit is entirely before this range + self.location += (Int(edit.newEndByte) - Int(edit.oldEndByte))/2 + } else if edit.startByte/2 < location { + // If the edit starts in the space before this range and extends into this range + length -= Int(edit.oldEndByte)/2 - location + location = Int(edit.newEndByte)/2 + } else if edit.startByte/2 == location && isPureInsertion { + // If the edit is *only* an insertion right at the beginning of the range + location = Int(edit.newEndByte)/2 + } else { + // Otherwise, the edit is entirely within this range + if edit.startByte/2 < endIndex || (edit.startByte/2 == endIndex && isPureInsertion) { + length = (Int(edit.newEndByte)/2 - location) + (length - (Int(edit.oldEndByte)/2 - location)) + } + } + } +} diff --git a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+TSRange.swift b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+TSRange.swift index 1b3f8a749..319763999 100644 --- a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+TSRange.swift +++ b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+TSRange.swift @@ -10,6 +10,7 @@ import SwiftTreeSitter extension NSRange { var tsRange: TSRange { - return TSRange(points: .zero..<(.zero), bytes: (UInt32(self.lowerBound) * 2)..<(UInt32(self.upperBound) * 2)) + return TSRange(points: .zero..<(.zero), + bytes: (UInt32(self.location) * 2)..<(UInt32(self.location + self.length) * 2)) } } diff --git a/Sources/CodeEditTextView/Highlighting/HighlightProviding.swift b/Sources/CodeEditTextView/Highlighting/HighlightProviding.swift index cd376ef9f..199c2cc42 100644 --- a/Sources/CodeEditTextView/Highlighting/HighlightProviding.swift +++ b/Sources/CodeEditTextView/Highlighting/HighlightProviding.swift @@ -17,6 +17,9 @@ public protocol HighlightProviding { /// - Note: This does not need to be *globally* unique, merely unique across all the highlighters used. var identifier: String { get } + /// Called once at editor initialization. + func setUp(textView: HighlighterTextView) + /// Updates the highlighter's code language. /// - Parameters: /// - codeLanguage: The langugage that should be used by the highlighter. diff --git a/Sources/CodeEditTextView/Highlighting/Highlighter.swift b/Sources/CodeEditTextView/Highlighting/Highlighter.swift index 83016e130..ebda49b97 100644 --- a/Sources/CodeEditTextView/Highlighting/Highlighter.swift +++ b/Sources/CodeEditTextView/Highlighting/Highlighter.swift @@ -88,6 +88,7 @@ class Highlighter: NSObject { } textView.textContentStorage.textStorage?.delegate = self + highlightProvider?.setUp(textView: textView) if let scrollView = textView.enclosingScrollView { NotificationCenter.default.addObserver(self, @@ -121,6 +122,7 @@ class Highlighter: NSObject { public func setHighlightProvider(_ provider: HighlightProviding) { self.highlightProvider = provider highlightProvider?.setLanguage(codeLanguage: language) + highlightProvider?.setUp(textView: textView) invalidate() } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift new file mode 100644 index 000000000..36ea52019 --- /dev/null +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift @@ -0,0 +1,86 @@ +// +// TreeSitterClient+Edit.swift +// +// +// Created by Khan Winter on 3/10/23. +// + +import Foundation +import SwiftTreeSitter +import CodeEditLanguages + +extension TreeSitterClient { + + /// Calculates a series of ranges that have been invalidated by a given edit. + /// - Parameters: + /// - textView: The text view to use for text. + /// - edit: The edit to act on. + /// - language: The language to use. + /// - readBlock: A callback for fetching blocks of text. + /// - Returns: An array of distinct `NSRanges` that need to be re-highlighted. + func findChangedByteRanges(textView: HighlighterTextView, + edit: InputEdit, + layer: LanguageLayer, + readBlock: @escaping Parser.ReadBlock) -> [NSRange] { + let (oldTree, newTree) = calculateNewState(tree: layer.tree, + parser: layer.parser, + edit: edit, + readBlock: readBlock) + if oldTree == nil && newTree == nil { + // There was no existing tree, make a new one and return all indexes. + layer.tree = createTree(parser: layer.parser, readBlock: readBlock) + return [NSRange(textView.documentRange.intRange)] + } + + let ranges = changedByteRanges(oldTree, rhs: newTree).map { $0.range } + + layer.tree = newTree + + return ranges + } + + /// Applies the edit to the current `tree` and returns the old tree and a copy of the current tree with the + /// processed edit. + /// - Parameters: + /// - tree: The tree before an edit used to parse the new tree. + /// - parser: The parser used to parse the new tree. + /// - edit: The edit to apply. + /// - readBlock: The block to use to read text. + /// - Returns: (The old state, the new state). + internal func calculateNewState(tree: Tree?, + parser: Parser, + edit: InputEdit, + readBlock: @escaping Parser.ReadBlock) -> (Tree?, Tree?) { + guard let oldTree = tree else { + return (nil, nil) + } + semaphore.wait() + + // Apply the edit to the old tree + oldTree.edit(edit) + + let newTree = parser.parse(tree: oldTree, readBlock: readBlock) + + semaphore.signal() + + return (oldTree.copy(), newTree) + } + + /// Calculates the changed byte ranges between two trees. + /// - Parameters: + /// - lhs: The first (older) tree. + /// - rhs: The second (newer) tree. + /// - Returns: Any changed ranges. + internal func changedByteRanges(_ lhs: Tree?, rhs: Tree?) -> [Range] { + switch (lhs, rhs) { + case (let t1?, let t2?): + return t1.changedRanges(from: t2).map({ $0.bytes }) + case (nil, let t2?): + let range = t2.rootNode?.byteRange + + return range.flatMap({ [$0] }) ?? [] + case (_, nil): + return [] + } + } +} diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift new file mode 100644 index 000000000..38fcce622 --- /dev/null +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift @@ -0,0 +1,145 @@ +// +// TreeSitterClient+Highlight.swift +// +// +// Created by Khan Winter on 3/10/23. +// + +import Foundation +import SwiftTreeSitter +import CodeEditLanguages + +extension TreeSitterClient { + + /// Queries the given language layer for any highlights. + /// - Parameters: + /// - layer: The layer to query. + /// - textView: A text view to use for contextual data. + /// - range: The range to query for. + /// - Returns: Any ranges to highlight. + internal func queryLayerHighlights(layer: LanguageLayer, + textView: HighlighterTextView, + range: NSRange) -> [HighlightRange] { + // Make sure we don't change the tree while we copy it. + self.semaphore.wait() + + guard let tree = layer.tree?.copy() else { + self.semaphore.signal() + return [] + } + + self.semaphore.signal() + + guard let rootNode = tree.rootNode else { + return [] + } + + // This needs to be on the main thread since we're going to use the `textProvider` in + // the `highlightsFromCursor` method, which uses the textView's text storage. + guard let cursor = layer.languageQuery?.execute(node: rootNode, in: tree) else { + return [] + } + cursor.setRange(range) + cursor.matchLimit = Constants.treeSitterMatchLimit + + return highlightsFromCursor(cursor: ResolvingQueryCursor(cursor: cursor)) + } + + /// Performs an injections query on the given language layer. + /// Updates any existing layers with new ranges and adds new layers if needed. + /// - Parameters: + /// - textView: The text view to use. + /// - language: The language layer to perform the query on. + /// - readBlock: A completion block for reading from text storage efficiently. + /// - Returns: An index set of any updated indexes. + @discardableResult + internal func updateInjectedLanguageLayers(textView: HighlighterTextView, + language: LanguageLayer, + readBlock: @escaping Parser.ReadBlock) -> IndexSet { + guard let tree = language.tree, + let rootNode = tree.rootNode, + let cursor = language.languageQuery?.execute(node: rootNode, in: tree) else { + return IndexSet() + } + + cursor.matchLimit = Constants.treeSitterMatchLimit + + let languageRanges = self.injectedLanguagesFrom(cursor: cursor) { range, _ in + return textView.stringForRange(range) + } + + var updatedRanges = IndexSet() + for (languageName, ranges) in languageRanges { + guard let treeSitterLanguage = TreeSitterLanguage(rawValue: languageName) else { + continue + } + + if treeSitterLanguage == primaryLayer { + continue + } + + // Add the language if not available + if let layerIndex = layers.firstIndex(where: { $0.id == treeSitterLanguage }) { + // Add any ranges not included in the layer already + for range in ranges where !layers[layerIndex].ranges.contains(range.range) { + updatedRanges.insert(range: range.range) + layers[layerIndex].ranges.append(range.range) + } + } else { + addLanguageLayer(layerId: treeSitterLanguage, readBlock: readBlock) + + let layerIndex = layers.count - 1 + guard layers.last?.id == treeSitterLanguage else { + continue + } + + layers[layerIndex].parser.includedRanges = ranges.map { $0.tsRange } + layers[layerIndex].ranges = ranges.map { $0.range } + } + } + return updatedRanges + } + + /// Resolves a query cursor to the highlight ranges it contains. + /// **Must be called on the main thread** + /// - Parameter cursor: The cursor to resolve. + /// - Returns: Any highlight ranges contained in the cursor. + internal func highlightsFromCursor(cursor: ResolvingQueryCursor) -> [HighlightRange] { + cursor.prepare(with: self.textProvider) + return cursor + .flatMap { $0.captures } + .compactMap { + // Some languages add an "@spell" capture to indicate a portion of text that should be spellchecked + // (usually comments). But this causes other captures in the same range to be overriden. So we ignore + // that specific capture type. + if $0.name != "spell" && $0.name != "injection.content" { + return HighlightRange(range: $0.range, capture: CaptureName.fromString($0.name ?? "")) + } + return nil + } + } + + /// Returns all injected languages from a given cursor. The cursor must be new, + /// having not been used for normal highlight matching. + /// - Parameters: + /// - cursor: The cursor to use for finding injected languages. + /// - textProvider: A callback for efficiently fetching text. + /// - Returns: A map of each language to all the ranges they have been injected into. + internal func injectedLanguagesFrom( + cursor: QueryCursor, + textProvider: @escaping ResolvingQueryCursor.TextProvider + ) -> [String: [NamedRange]] { + var languages: [String: [NamedRange]] = [:] + + for match in cursor { + if let injection = match.injection(with: textProvider) { + if languages[injection.name] == nil { + languages[injection.name] = [] + } + languages[injection.name]?.append(injection) + } + } + + return languages + } +} diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift new file mode 100644 index 000000000..d4a9f766f --- /dev/null +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift @@ -0,0 +1,33 @@ +// +// TreeSitterClient+LanguageLayer.swift +// +// +// Created by Khan Winter on 3/8/23. +// + +import Foundation +import CodeEditLanguages +import SwiftTreeSitter + +extension TreeSitterClient { + class LanguageLayer { + init(id: TreeSitterLanguage, + parser: Parser, + tree: Tree? = nil, + languageQuery: Query? = nil, + ranges: [NSRange]) { + self.id = id + self.parser = parser + self.tree = tree + self.languageQuery = languageQuery + self.ranges = ranges + } + + var id: TreeSitterLanguage + var parser: Parser + var tree: Tree? + var languageQuery: Query? + var ranges: [NSRange] + var color: CaptureName = [.string, .boolean, .number].randomElement()! + } +} diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index a382f33ef..b7c16228a 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -18,66 +18,88 @@ import SwiftTreeSitter /// However, the `setText` method will re-compile the entire corpus so should be used sparingly. final class TreeSitterClient: HighlightProviding { + // MARK: - Properties/Constants + public var identifier: String { "CodeEdit.TreeSitterClient" } - private var primaryLanguage: TreeSitterLanguage - private var languages: [TreeSitterLanguage: Language] = [:] - - class Language { - init(id: TreeSitterLanguage, - parser: Parser, - tree: Tree? = nil, - languageQuery: Query? = nil) { - self.id = id - self.parser = parser - self.tree = tree - self.languageQuery = languageQuery - } - - var id: TreeSitterLanguage - var parser: Parser - var tree: Tree? - var languageQuery: Query? - } + internal var primaryLayer: TreeSitterLanguage + internal var layers: [LanguageLayer] = [] - private var textProvider: ResolvingQueryCursor.TextProvider + internal var textProvider: ResolvingQueryCursor.TextProvider /// The queue to do tree-sitter work on for large edits/queries - private let queue: DispatchQueue = DispatchQueue(label: "CodeEdit.CodeEditTextView.TreeSitter", - qos: .userInteractive) + internal let queue: DispatchQueue = DispatchQueue(label: "CodeEdit.CodeEditTextView.TreeSitter", + qos: .userInteractive) /// Used to ensure safe use of the shared tree-sitter tree state in different sync/async contexts. - private let semaphore: DispatchSemaphore = DispatchSemaphore(value: 1) + internal let semaphore: DispatchSemaphore = DispatchSemaphore(value: 1) + + internal enum Constants { + /// The maximum amount of limits a cursor can match during a query. + /// Used to ensure performance in large files, even though we generally limit the query to the visible range. + /// Neovim encountered this issue and uses 64 for their limit. Helix uses 256 due to issues with some + /// languages when using 64. + /// See: https://github.com/neovim/neovim/issues/14897 + /// And: https://github.com/helix-editor/helix/pull/4830 + static let treeSitterMatchLimit = 256 + } + + // MARK: - Init/Config /// Initializes the `TreeSitterClient` with the given parameters. /// - Parameters: /// - codeLanguage: The language to set up the parser with. /// - textProvider: The text provider callback to read any text. public init(codeLanguage: CodeLanguage, textProvider: @escaping ResolvingQueryCursor.TextProvider) throws { - primaryLanguage = codeLanguage.id - languages[codeLanguage.id] = Language(id: codeLanguage.id, - parser: Parser(), - tree: nil, - languageQuery: TreeSitterModel.shared.query(for: codeLanguage.id)) + primaryLayer = codeLanguage.id + layers = [ + LanguageLayer(id: codeLanguage.id, + parser: Parser(), + tree: nil, + languageQuery: TreeSitterModel.shared.query(for: codeLanguage.id), + ranges: []) + ] self.textProvider = textProvider if let treeSitterLanguage = codeLanguage.language { - try languages[codeLanguage.id]?.parser.setLanguage(treeSitterLanguage) + try? layers[0].parser.setLanguage(treeSitterLanguage) } } - func setLanguage(codeLanguage: CodeLanguage) { + public func setLanguage(codeLanguage: CodeLanguage) { // Remove all trees and languages, everything needs to be re-parsed. - for key in languages.keys where key != codeLanguage.id { - languages.removeValue(forKey: key) - } - primaryLanguage = codeLanguage.id + layers.removeAll() + + primaryLayer = codeLanguage.id + layers = [ + LanguageLayer(id: codeLanguage.id, + parser: Parser(), + tree: nil, + languageQuery: TreeSitterModel.shared.query(for: codeLanguage.id), + ranges: []) + ] if let treeSitterLanguage = codeLanguage.language { - try? languages[codeLanguage.id]?.parser.setLanguage(treeSitterLanguage) + try? layers[0].parser.setLanguage(treeSitterLanguage) + } + } + + // MARK: - HighlightProviding + + /// Set up and parse the initial language tree + func setUp(textView: HighlighterTextView) { + let readBlock = createReadBlock(textView: textView) + + layers[0].tree = createTree(parser: layers[0].parser, + readBlock: readBlock) + + for layer in layers { + updateInjectedLanguageLayers(textView: textView, + language: layer, + readBlock: readBlock) } } @@ -97,47 +119,35 @@ final class TreeSitterClient: HighlightProviding { } let readBlock = createReadBlock(textView: textView) + var rangeSet = IndexSet() - let effectedRanges: [NSRange] = findChangedByteRanges( - textView: textView, - edit: edit, - language: languages[primaryLanguage]!, - readBlock: readBlock - ) + for layer in layers { + if layer.id != primaryLayer { + for idx in (0.. [NSRange] { - let (oldTree, newTree) = calculateNewState(tree: language.tree, - parser: language.parser, - edit: edit, - readBlock: readBlock) - if oldTree == nil && newTree == nil { - // There was no existing tree, make a new one and return all indexes. - languages[language.id]?.tree = createTree(parser: language.parser, readBlock: readBlock) - return [NSRange(textView.documentRange.intRange)] + for layer in layers { + rangeSet.formUnion( + updateInjectedLanguageLayers(textView: textView, + language: layer, + readBlock: readBlock) + ) } - - let ranges = changedByteRanges(oldTree, rhs: newTree).map { $0.range } - - languages[language.id]?.tree = newTree - - return ranges + completion(rangeSet) } /// Initiates a highlight query. @@ -147,135 +157,62 @@ final class TreeSitterClient: HighlightProviding { /// - completion: Called when the query completes. func queryHighlightsFor(textView: HighlighterTextView, range: NSRange, - completion: @escaping (([HighlightRange]) -> Void)) { - let readBlock = createReadBlock(textView: textView) + completion: @escaping ((([HighlightRange]) -> Void))) { + var highlights: [(TreeSitterLanguage, HighlightRange)] = [] + var injectedSet = IndexSet(integersIn: range) + + for layer in layers where layer.id != primaryLayer { + // Query injected only if a layer's ranges intersects with `range` + for layerRange in layer.ranges + where layerRange.location <= NSMaxRange(range) && range.location <= NSMaxRange(layerRange) { + let location = max(layerRange.location, range.location) + let length = min(NSMaxRange(layerRange), NSMaxRange(range)) - location + let rangeIntersection = NSRange(location: location, + length: length) + highlights.append( + contentsOf: queryLayerHighlights(layer: layer, + textView: textView, + range: rangeIntersection).map { (layer.id, $0) } + ) - let language = languages[primaryLanguage]! - // Make sure we dont accidentally change the tree while we copy it. - self.semaphore.wait() - guard let tree = language.tree?.copy() else { - // In this case, we don't have a tree to work with already, so we need to make it and try to - // return some highlights - language.tree = createTree(parser: language.parser, readBlock: readBlock) - - // This is slightly redundant but we're only doing one check. - guard let treeRetry = language.tree?.copy() else { - // Now we can return nothing for real. - self.semaphore.signal() - completion([]) - return + injectedSet.remove(integersIn: rangeIntersection) } - self.semaphore.signal() - - completion( - _queryColorsFor(textView: textView, language: language, tree: treeRetry, range: range) + - queryInjectedLanguages(textView: textView, - language: language, - tree: treeRetry, - range: range, - readBlock: readBlock) - ) - return - } - - self.semaphore.signal() - - completion( - _queryColorsFor(textView: textView, language: language, tree: tree, range: range) + - queryInjectedLanguages(textView: textView, - language: language, - tree: tree, - range: range, - readBlock: readBlock) - ) - } - - private func _queryColorsFor(textView: HighlighterTextView, - language: Language, - tree: Tree, - range: NSRange) -> [HighlightRange] { - guard let rootNode = tree.rootNode else { - return [] } - // This needs to be on the main thread since we're going to use the `textProvider` in - // the `highlightsFromCursor` method, which uses the textView's text storage. - guard let cursor = language.languageQuery?.execute(node: rootNode, in: tree) else { - return [] + // Query primary for any ranges that weren't used in the injected layers. + for range in injectedSet.rangeView { + highlights.append(contentsOf: queryLayerHighlights(layer: layers[0], + textView: textView, + range: NSRange(range)).map { (layers[0].id, $0) }) } - cursor.setRange(range) - - let highlights = highlightsFromCursor(cursor: ResolvingQueryCursor(cursor: cursor)) - return highlights + completion(highlights.map { $0.1 }) } - private func queryInjectedLanguages(textView: HighlighterTextView, - language: Language, - tree: Tree, - range: NSRange, - readBlock: @escaping Parser.ReadBlock) -> [HighlightRange] { - guard let rootNode = tree.rootNode, - let cursor = language.languageQuery?.execute(node: rootNode, in: tree) else { - return [] - } - - cursor.setRange(range) - - let languageRanges = self.injectedLanguagesFrom(cursor: cursor) { range, _ in - return textView.stringForRange(range) + // MARK: - Helpers + + /// Attempts to add a language to the `languages` dictionary. + /// - Parameter language: The language to add. + internal func addLanguageLayer(layerId: TreeSitterLanguage, readBlock: @escaping Parser.ReadBlock) { + let newLayer = LanguageLayer(id: layerId, + parser: Parser(), + tree: nil, + languageQuery: TreeSitterModel.shared.query(for: layerId), + ranges: []) + + guard let parserLanguage = CodeLanguage + .allLanguages + .first(where: { $0.id == layerId })? + .language + else { + return } - var highlights: [HighlightRange] = [] - - for (languageName, ranges) in languageRanges { - guard let language = TreeSitterLanguage(rawValue: languageName) else { - continue - } - - if language == primaryLanguage { - continue - } - - if languages[language] == nil { - languages[language] = Language(id: language, - parser: Parser(), - tree: nil, - languageQuery: TreeSitterModel.shared.query(for: language)) - } - - guard let parserLanguage = CodeLanguage - .allLanguages - .first(where: { $0.id == language })? - .language - else { - continue - } - - try? languages[language]?.parser.setLanguage(parserLanguage) - languages[language]?.parser.includedRanges = ranges.map { $0.tsRange } - languages[language]?.tree = createTree(parser: languages[language]!.parser, - readBlock: readBlock) - - for range in ranges { - highlights.append( - contentsOf: _queryColorsFor(textView: textView, - language: languages[language]!, - tree: languages[language]!.tree!, - range: range.range) - ) - } - - highlights.append( - contentsOf: queryInjectedLanguages(textView: textView, - language: languages[language]!, - tree: languages[language]!.tree!, - range: range, - readBlock: readBlock) - ) - } + try? newLayer.parser.setLanguage(parserLanguage) + newLayer.tree = createTree(parser: newLayer.parser, + readBlock: readBlock) - return highlights + layers.append(newLayer) } /// Creates a tree-sitter tree. @@ -283,50 +220,11 @@ final class TreeSitterClient: HighlightProviding { /// - parser: The parser object to use to parse text. /// - readBlock: A callback for fetching blocks of text. /// - Returns: A tree if it could be parsed. - private func createTree(parser: Parser, readBlock: @escaping Parser.ReadBlock) -> Tree? { + internal func createTree(parser: Parser, readBlock: @escaping Parser.ReadBlock) -> Tree? { return parser.parse(tree: nil, readBlock: readBlock) } - /// Resolves a query cursor to the highlight ranges it contains. - /// **Must be called on the main thread** - /// - Parameter cursor: The cursor to resolve. - /// - Returns: Any highlight ranges contained in the cursor. - private func highlightsFromCursor(cursor: ResolvingQueryCursor) -> [HighlightRange] { - cursor.prepare(with: self.textProvider) - return cursor - .flatMap { $0.captures } - .map { - HighlightRange(range: $0.range, capture: CaptureName.fromString($0.name ?? "")) - } - } - - /// Returns all injected languages from a given cursor. The cursor must be new, - /// having not been used for normal highlight matching. - /// - Parameters: - /// - cursor: The cursor to use for finding injected languages. - /// - textProvider: A callback for efficiently fetching text. - /// - Returns: A map of each language to all the ranges they have been injected into. - private func injectedLanguagesFrom( - cursor: QueryCursor, - textProvider: @escaping ResolvingQueryCursor.TextProvider - ) -> [String: [NamedRange]] { - var languages: [String: [NamedRange]] = [:] - - for match in cursor { - if let injection = match.injection(with: textProvider) { - if languages[injection.name] == nil { - languages[injection.name] = [] - } - languages[injection.name]?.append(injection) - } - } - - return languages - } -} - -extension TreeSitterClient { - private func createReadBlock(textView: HighlighterTextView) -> Parser.ReadBlock { + internal func createReadBlock(textView: HighlighterTextView) -> Parser.ReadBlock { return { byteOffset, _ in let limit = textView.documentRange.length let location = byteOffset / 2 @@ -339,47 +237,4 @@ extension TreeSitterClient { return textView.stringForRange(range)?.data(using: String.nativeUTF16Encoding) } } - - /// Applies the edit to the current `tree` and returns the old tree and a copy of the current tree with the - /// processed edit. - /// - Parameters: - /// - edit: The edit to apply. - /// - readBlock: The block to use to read text. - /// - Returns: (The old state, the new state). - private func calculateNewState(tree: Tree?, - parser: Parser, - edit: InputEdit, - readBlock: @escaping Parser.ReadBlock) -> (Tree?, Tree?) { - guard let oldTree = tree else { - return (nil, nil) - } - semaphore.wait() - - // Apply the edit to the old tree - oldTree.edit(edit) - - let newTree = parser.parse(tree: oldTree, readBlock: readBlock) - - semaphore.signal() - - return (oldTree.copy(), newTree) - } - - /// Calculates the changed byte ranges between two trees. - /// - Parameters: - /// - lhs: The first (older) tree. - /// - rhs: The second (newer) tree. - /// - Returns: Any changed ranges. - private func changedByteRanges(_ lhs: Tree?, rhs: Tree?) -> [Range] { - switch (lhs, rhs) { - case (let t1?, let t2?): - return t1.changedRanges(from: t2).map({ $0.bytes }) - case (nil, let t2?): - let range = t2.rootNode?.byteRange - - return range.flatMap({ [$0] }) ?? [] - case (_, nil): - return [] - } - } } From 546f16d28425e110d9458927c309cc22cf566dd8 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Tue, 14 Mar 2023 20:01:53 -0500 Subject: [PATCH 05/10] Initial highlights work & remove debugs --- .../TreeSitterClient+Highlight.swift | 12 ++-- .../TreeSitterClient+LanguageLayer.swift | 1 - .../TreeSitter/TreeSitterClient.swift | 55 ++++++++++--------- 3 files changed, 36 insertions(+), 32 deletions(-) diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift index 38fcce622..5f5ea0bf7 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift @@ -78,14 +78,15 @@ extension TreeSitterClient { continue } - // Add the language if not available - if let layerIndex = layers.firstIndex(where: { $0.id == treeSitterLanguage }) { + if let layer = layers.first(where: { $0.id == treeSitterLanguage }) { // Add any ranges not included in the layer already - for range in ranges where !layers[layerIndex].ranges.contains(range.range) { - updatedRanges.insert(range: range.range) - layers[layerIndex].ranges.append(range.range) + for namedRange in ranges + where !layer.ranges.contains(where: { $0.intersection(namedRange.range) != nil }) { + updatedRanges.insert(range: namedRange.range) + layer.ranges.append(namedRange.range) } } else { + // Add the language if not available addLanguageLayer(layerId: treeSitterLanguage, readBlock: readBlock) let layerIndex = layers.count - 1 @@ -95,6 +96,7 @@ extension TreeSitterClient { layers[layerIndex].parser.includedRanges = ranges.map { $0.tsRange } layers[layerIndex].ranges = ranges.map { $0.range } + layers[layerIndex].tree = createTree(parser: layers[layerIndex].parser, readBlock: readBlock) } } return updatedRanges diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift index d4a9f766f..14caa52d2 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift @@ -28,6 +28,5 @@ extension TreeSitterClient { var tree: Tree? var languageQuery: Query? var ranges: [NSRange] - var color: CaptureName = [.string, .boolean, .number].randomElement()! } } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index b7c16228a..e718c9928 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -53,22 +53,13 @@ final class TreeSitterClient: HighlightProviding { /// - codeLanguage: The language to set up the parser with. /// - textProvider: The text provider callback to read any text. public init(codeLanguage: CodeLanguage, textProvider: @escaping ResolvingQueryCursor.TextProvider) throws { - primaryLayer = codeLanguage.id - layers = [ - LanguageLayer(id: codeLanguage.id, - parser: Parser(), - tree: nil, - languageQuery: TreeSitterModel.shared.query(for: codeLanguage.id), - ranges: []) - ] - self.textProvider = textProvider - - if let treeSitterLanguage = codeLanguage.language { - try? layers[0].parser.setLanguage(treeSitterLanguage) - } + self.primaryLayer = codeLanguage.id + setLanguage(codeLanguage: codeLanguage) } + /// Sets the primary language for the client. Will reset all layers, will not do any parsing work. + /// - Parameter codeLanguage: The new primary language. public func setLanguage(codeLanguage: CodeLanguage) { // Remove all trees and languages, everything needs to be re-parsed. layers.removeAll() @@ -89,17 +80,24 @@ final class TreeSitterClient: HighlightProviding { // MARK: - HighlightProviding - /// Set up and parse the initial language tree + /// Set up and parse the initial language tree and all injected layers. func setUp(textView: HighlighterTextView) { let readBlock = createReadBlock(textView: textView) layers[0].tree = createTree(parser: layers[0].parser, readBlock: readBlock) - for layer in layers { + var idx = 0 + while idx < layers.count { + if layers[idx].id != primaryLayer { + layers[idx].parser.includedRanges = layers[idx].ranges.map { $0.tsRange } + layers[idx].tree = createTree(parser: layers[idx].parser, readBlock: readBlock) + } updateInjectedLanguageLayers(textView: textView, - language: layer, + language: layers[idx], readBlock: readBlock) + + idx += 1 } } @@ -121,12 +119,17 @@ final class TreeSitterClient: HighlightProviding { let readBlock = createReadBlock(textView: textView) var rangeSet = IndexSet() - for layer in layers { + // Loop through all layers and apply the edit, and query for any changed byte ranges. + // Using a while loop b/c `updateInjectedLanguageLayers` can append new layers during the loop. + var idx = 0 + while idx < layers.count { + let layer = layers[idx] + // The primary layer's range is always the entire document, no need to modify. if layer.id != primaryLayer { for idx in (0.. Void))) { - var highlights: [(TreeSitterLanguage, HighlightRange)] = [] + var highlights: [HighlightRange] = [] var injectedSet = IndexSet(integersIn: range) for layer in layers where layer.id != primaryLayer { @@ -172,7 +177,7 @@ final class TreeSitterClient: HighlightProviding { highlights.append( contentsOf: queryLayerHighlights(layer: layer, textView: textView, - range: rangeIntersection).map { (layer.id, $0) } + range: rangeIntersection) ) injectedSet.remove(integersIn: rangeIntersection) @@ -183,10 +188,10 @@ final class TreeSitterClient: HighlightProviding { for range in injectedSet.rangeView { highlights.append(contentsOf: queryLayerHighlights(layer: layers[0], textView: textView, - range: NSRange(range)).map { (layers[0].id, $0) }) + range: NSRange(range))) } - completion(highlights.map { $0.1 }) + completion(highlights) } // MARK: - Helpers @@ -209,8 +214,6 @@ final class TreeSitterClient: HighlightProviding { } try? newLayer.parser.setLanguage(parserLanguage) - newLayer.tree = createTree(parser: newLayer.parser, - readBlock: readBlock) layers.append(newLayer) } From abb3ade87993a8c99490893e3cb3a148ccded2ed Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Wed, 15 Mar 2023 22:50:29 -0500 Subject: [PATCH 06/10] Fix editing & new injections --- .../NSRange+/NSRange+Comparable.swift | 18 +++++ .../NSRange+/NSRange+InputEdit.swift | 34 --------- .../TreeSitter/TreeSitterClient+Edit.swift | 74 +++++++++++++++++++ .../TreeSitterClient+Highlight.swift | 57 -------------- .../TreeSitterClient+LanguageLayer.swift | 4 +- .../TreeSitter/TreeSitterClient.swift | 18 +---- 6 files changed, 96 insertions(+), 109 deletions(-) create mode 100644 Sources/CodeEditTextView/Extensions/NSRange+/NSRange+Comparable.swift diff --git a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+Comparable.swift b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+Comparable.swift new file mode 100644 index 000000000..662287ef9 --- /dev/null +++ b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+Comparable.swift @@ -0,0 +1,18 @@ +// +// File.swift +// +// +// Created by Khan Winter on 3/15/23. +// + +import Foundation + +extension NSRange: Comparable { + public static func == (lhs: NSRange, rhs: NSRange) -> Bool { + return lhs.location == rhs.location && lhs.length == rhs.length + } + + public static func < (lhs: NSRange, rhs: NSRange) -> Bool { + return lhs.location < rhs.location + } +} diff --git a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift index b2b97f0e2..9994a92df 100644 --- a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift +++ b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift @@ -29,37 +29,3 @@ extension InputEdit { newEndPoint: newEndPoint) } } - -extension NSRange { - // swiftlint:disable line_length - - /// Modifies the range to account for an edit. - /// Largely based on: - /// [tree-sitter](https://github.com/tree-sitter/tree-sitter/blob/ddeaa0c7f534268b35b4f6cb39b52df082754413/lib/src/subtree.c#L691-L720) - mutating func applyInputEdit(_ edit: InputEdit) { - // swiftlint:enable line_length - - let endIndex = NSMaxRange(self) - let isPureInsertion = edit.oldEndByte == edit.startByte - - // Edit is after the range - if (edit.startByte/2) > endIndex { - return - } else if edit.oldEndByte/2 < location { - // If the edit is entirely before this range - self.location += (Int(edit.newEndByte) - Int(edit.oldEndByte))/2 - } else if edit.startByte/2 < location { - // If the edit starts in the space before this range and extends into this range - length -= Int(edit.oldEndByte)/2 - location - location = Int(edit.newEndByte)/2 - } else if edit.startByte/2 == location && isPureInsertion { - // If the edit is *only* an insertion right at the beginning of the range - location = Int(edit.newEndByte)/2 - } else { - // Otherwise, the edit is entirely within this range - if edit.startByte/2 < endIndex || (edit.startByte/2 == endIndex && isPureInsertion) { - length = (Int(edit.newEndByte)/2 - location) + (length - (Int(edit.oldEndByte)/2 - location)) - } - } - } -} diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift index 36ea52019..24032fff3 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift @@ -83,4 +83,78 @@ extension TreeSitterClient { return [] } } + + /// Performs an injections query on the given language layer. + /// Updates any existing layers with new ranges and adds new layers if needed. + /// - Parameters: + /// - textView: The text view to use. + /// - language: The language layer to perform the query on. + /// - readBlock: A completion block for reading from text storage efficiently. + /// - Returns: An index set of any updated indexes. + @discardableResult + internal func updateInjectedLanguageLayers(textView: HighlighterTextView, + language: LanguageLayer, + readBlock: @escaping Parser.ReadBlock) -> IndexSet { + guard let tree = language.tree, + let rootNode = tree.rootNode, + let cursor = language.languageQuery?.execute(node: rootNode, in: tree) else { + return IndexSet() + } + + cursor.matchLimit = Constants.treeSitterMatchLimit + + let languageRanges = self.injectedLanguagesFrom(cursor: cursor) { range, _ in + return textView.stringForRange(range) + } + + var updatedRanges = IndexSet() + for (languageName, ranges) in languageRanges { + guard let treeSitterLanguage = TreeSitterLanguage(rawValue: languageName) else { + continue + } + + if treeSitterLanguage == primaryLayer { + continue + } + + if let layer = layers.first(where: { $0.id == treeSitterLanguage }) { + // Add any ranges not included in the layer already + // and update any overlapping ones. + for namedRange in ranges where namedRange.range.length > 0 { + var wasRangeFound = false + + for (idx, layerRange) in layer.ranges.enumerated().reversed() + where namedRange.range.intersection(layerRange) != nil { + wasRangeFound = true + layer.ranges[idx] = namedRange.range + break + } + + if !wasRangeFound { + layer.ranges.append(namedRange.range) + updatedRanges.insert(range: namedRange.range) + } + } + + // Required for tree-sitter to work. Assumes no ranges are overlapping. + layer.ranges.sort() + } else { + // Add the language if not available + addLanguageLayer(layerId: treeSitterLanguage, readBlock: readBlock) + + let layerIndex = layers.count - 1 + guard layers.last?.id == treeSitterLanguage else { + continue + } + + layers[layerIndex].parser.includedRanges = ranges + .filter { $0.range.length > 0 } + .map { $0.tsRange } + .sorted() + layers[layerIndex].ranges = ranges.filter { $0.range.length > 0 }.map { $0.range } + layers[layerIndex].tree = createTree(parser: layers[layerIndex].parser, readBlock: readBlock) + } + } + return updatedRanges + } } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift index 5f5ea0bf7..8b1198029 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift @@ -45,63 +45,6 @@ extension TreeSitterClient { return highlightsFromCursor(cursor: ResolvingQueryCursor(cursor: cursor)) } - /// Performs an injections query on the given language layer. - /// Updates any existing layers with new ranges and adds new layers if needed. - /// - Parameters: - /// - textView: The text view to use. - /// - language: The language layer to perform the query on. - /// - readBlock: A completion block for reading from text storage efficiently. - /// - Returns: An index set of any updated indexes. - @discardableResult - internal func updateInjectedLanguageLayers(textView: HighlighterTextView, - language: LanguageLayer, - readBlock: @escaping Parser.ReadBlock) -> IndexSet { - guard let tree = language.tree, - let rootNode = tree.rootNode, - let cursor = language.languageQuery?.execute(node: rootNode, in: tree) else { - return IndexSet() - } - - cursor.matchLimit = Constants.treeSitterMatchLimit - - let languageRanges = self.injectedLanguagesFrom(cursor: cursor) { range, _ in - return textView.stringForRange(range) - } - - var updatedRanges = IndexSet() - for (languageName, ranges) in languageRanges { - guard let treeSitterLanguage = TreeSitterLanguage(rawValue: languageName) else { - continue - } - - if treeSitterLanguage == primaryLayer { - continue - } - - if let layer = layers.first(where: { $0.id == treeSitterLanguage }) { - // Add any ranges not included in the layer already - for namedRange in ranges - where !layer.ranges.contains(where: { $0.intersection(namedRange.range) != nil }) { - updatedRanges.insert(range: namedRange.range) - layer.ranges.append(namedRange.range) - } - } else { - // Add the language if not available - addLanguageLayer(layerId: treeSitterLanguage, readBlock: readBlock) - - let layerIndex = layers.count - 1 - guard layers.last?.id == treeSitterLanguage else { - continue - } - - layers[layerIndex].parser.includedRanges = ranges.map { $0.tsRange } - layers[layerIndex].ranges = ranges.map { $0.range } - layers[layerIndex].tree = createTree(parser: layers[layerIndex].parser, readBlock: readBlock) - } - } - return updatedRanges - } - /// Resolves a query cursor to the highlight ranges it contains. /// **Must be called on the main thread** /// - Parameter cursor: The cursor to resolve. diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift index 14caa52d2..db4e70184 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift @@ -23,8 +23,8 @@ extension TreeSitterClient { self.ranges = ranges } - var id: TreeSitterLanguage - var parser: Parser + let id: TreeSitterLanguage + let parser: Parser var tree: Tree? var languageQuery: Query? var ranges: [NSRange] diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index e718c9928..da4028a93 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -52,7 +52,7 @@ final class TreeSitterClient: HighlightProviding { /// - Parameters: /// - codeLanguage: The language to set up the parser with. /// - textProvider: The text provider callback to read any text. - public init(codeLanguage: CodeLanguage, textProvider: @escaping ResolvingQueryCursor.TextProvider) throws { + public init(codeLanguage: CodeLanguage, textProvider: @escaping ResolvingQueryCursor.TextProvider) { self.textProvider = textProvider self.primaryLayer = codeLanguage.id setLanguage(codeLanguage: codeLanguage) @@ -89,10 +89,6 @@ final class TreeSitterClient: HighlightProviding { var idx = 0 while idx < layers.count { - if layers[idx].id != primaryLayer { - layers[idx].parser.includedRanges = layers[idx].ranges.map { $0.tsRange } - layers[idx].tree = createTree(parser: layers[idx].parser, readBlock: readBlock) - } updateInjectedLanguageLayers(textView: textView, language: layers[idx], readBlock: readBlock) @@ -124,16 +120,6 @@ final class TreeSitterClient: HighlightProviding { var idx = 0 while idx < layers.count { let layer = layers[idx] - // The primary layer's range is always the entire document, no need to modify. - if layer.id != primaryLayer { - for idx in (0.. Date: Fri, 24 Mar 2023 14:51:05 -0500 Subject: [PATCH 07/10] Switch to double pass & set of layers --- Package.swift | 2 +- .../Controller/STTextViewController.swift | 2 +- .../NSRange+/NSRange+InputEdit.swift | 32 ++++ .../Extensions/Tree+prettyPrint.swift | 71 +++++++++ .../TreeSitter/TreeSitterClient+Edit.swift | 69 ++++----- .../TreeSitterClient+LanguageLayer.swift | 22 ++- .../TreeSitter/TreeSitterClient.swift | 146 ++++++++++++------ 7 files changed, 254 insertions(+), 90 deletions(-) create mode 100644 Sources/CodeEditTextView/Extensions/Tree+prettyPrint.swift diff --git a/Package.swift b/Package.swift index 216a55eba..5465e69cd 100644 --- a/Package.swift +++ b/Package.swift @@ -19,7 +19,7 @@ let package = Package( ), .package( url: "https://github.com/CodeEditApp/CodeEditLanguages.git", - exact: "0.1.12" + from: "0.1.13" ), .package( url: "https://github.com/lukepistrol/SwiftLintPlugin", diff --git a/Sources/CodeEditTextView/Controller/STTextViewController.swift b/Sources/CodeEditTextView/Controller/STTextViewController.swift index 0dbee4f5a..1a1a9fb61 100644 --- a/Sources/CodeEditTextView/Controller/STTextViewController.swift +++ b/Sources/CodeEditTextView/Controller/STTextViewController.swift @@ -316,7 +316,7 @@ public class STTextViewController: NSViewController, STTextViewDelegate, ThemeAt return self?.textView.textContentStorage.textStorage?.mutableString.substring(with: range) } - provider = try? TreeSitterClient(codeLanguage: language, textProvider: textProvider) + provider = TreeSitterClient(codeLanguage: language, textProvider: textProvider) } if let provider = provider { diff --git a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift index 9994a92df..ee8018de5 100644 --- a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift +++ b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+InputEdit.swift @@ -29,3 +29,35 @@ extension InputEdit { newEndPoint: newEndPoint) } } + +extension NSRange { + // swiftlint:disable line_length + /// Modifies the range to account for an edit. + /// Largely based on code from + /// [tree-sitter](https://github.com/tree-sitter/tree-sitter/blob/ddeaa0c7f534268b35b4f6cb39b52df082754413/lib/src/subtree.c#L691-L720) + mutating func applyInputEdit(_ edit: InputEdit) { + // swiftlint:enable line_length + let endIndex = NSMaxRange(self) + let isPureInsertion = edit.oldEndByte == edit.startByte + + // Edit is after the range + if (edit.startByte/2) > endIndex { + return + } else if edit.oldEndByte/2 < location { + // If the edit is entirely before this range + self.location += (Int(edit.newEndByte) - Int(edit.oldEndByte))/2 + } else if edit.startByte/2 < location { + // If the edit starts in the space before this range and extends into this range + length -= Int(edit.oldEndByte)/2 - location + location = Int(edit.newEndByte)/2 + } else if edit.startByte/2 == location && isPureInsertion { + // If the edit is *only* an insertion right at the beginning of the range + location = Int(edit.newEndByte)/2 + } else { + // Otherwise, the edit is entirely within this range + if edit.startByte/2 < endIndex || (edit.startByte/2 == endIndex && isPureInsertion) { + length = (Int(edit.newEndByte)/2 - location) + (length - (Int(edit.oldEndByte)/2 - location)) + } + } + } +} diff --git a/Sources/CodeEditTextView/Extensions/Tree+prettyPrint.swift b/Sources/CodeEditTextView/Extensions/Tree+prettyPrint.swift new file mode 100644 index 000000000..a021733c9 --- /dev/null +++ b/Sources/CodeEditTextView/Extensions/Tree+prettyPrint.swift @@ -0,0 +1,71 @@ +// +// Tree+prettyPrint.swift +// +// +// Created by Khan Winter on 3/16/23. +// + +import SwiftTreeSitter + +#if DEBUG +extension Tree { + func prettyPrint() { + guard let cursor = self.rootNode?.treeCursor else { + print("NO ROOT NODE") + return + } + guard cursor.currentNode != nil else { + print("NO CURRENT NODE") + return + } + + func p(_ cursor: TreeCursor, depth: Int) { + guard let node = cursor.currentNode else { + return + } + + let visible = node.isNamed + + if visible { + print(String(repeating: " ", count: depth * 2), terminator: "") + if let fieldName = cursor.currentFieldName { + print(fieldName, ": ", separator: "", terminator: "") + } + print("(", node.nodeType ?? "NONE", " ", node.range, " ", separator: "", terminator: "") + } + + if cursor.goToFirstChild() { + while true { + if cursor.currentNode != nil && cursor.currentNode!.isNamed { + print("") + } + + p(cursor, depth: depth + 1) + + if !cursor.gotoNextSibling() { + break + } + } + + if !cursor.gotoParent() { + fatalError("Could not go to parent, this tree may be invalid.") + } + } + + if visible { + print(")", terminator: "") + } + } + + if cursor.currentNode?.childCount == 0 { + if !cursor.currentNode!.isNamed { + print("{\(cursor.currentNode!.nodeType ?? "NONE")}") + } else { + print("\"\(cursor.currentNode!.nodeType ?? "NONE")\"") + } + } else { + p(cursor, depth: 1) + } + } +} +#endif diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift index 24032fff3..fa706c594 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift @@ -88,16 +88,22 @@ extension TreeSitterClient { /// Updates any existing layers with new ranges and adds new layers if needed. /// - Parameters: /// - textView: The text view to use. - /// - language: The language layer to perform the query on. + /// - layer: The language layer to perform the query on. + /// - layerSet: The set of layers that exist in the document. + /// Used for efficient lookup of existing `(language, range)` pairs + /// - touchedLayers: The set of layers that existed before updating injected layers. + /// Will have items removed as they are found. /// - readBlock: A completion block for reading from text storage efficiently. /// - Returns: An index set of any updated indexes. @discardableResult internal func updateInjectedLanguageLayers(textView: HighlighterTextView, - language: LanguageLayer, + layer: LanguageLayer, + layerSet: inout Set, + touchedLayers: inout Set, readBlock: @escaping Parser.ReadBlock) -> IndexSet { - guard let tree = language.tree, + guard let tree = layer.tree, let rootNode = tree.rootNode, - let cursor = language.languageQuery?.execute(node: rootNode, in: tree) else { + let cursor = layer.languageQuery?.execute(node: rootNode, in: tree) else { return IndexSet() } @@ -108,6 +114,7 @@ extension TreeSitterClient { } var updatedRanges = IndexSet() + for (languageName, ranges) in languageRanges { guard let treeSitterLanguage = TreeSitterLanguage(rawValue: languageName) else { continue @@ -117,44 +124,30 @@ extension TreeSitterClient { continue } - if let layer = layers.first(where: { $0.id == treeSitterLanguage }) { - // Add any ranges not included in the layer already - // and update any overlapping ones. - for namedRange in ranges where namedRange.range.length > 0 { - var wasRangeFound = false - - for (idx, layerRange) in layer.ranges.enumerated().reversed() - where namedRange.range.intersection(layerRange) != nil { - wasRangeFound = true - layer.ranges[idx] = namedRange.range - break - } - - if !wasRangeFound { - layer.ranges.append(namedRange.range) - updatedRanges.insert(range: namedRange.range) + for range in ranges { + // Temp layer object for + let layer = LanguageLayer(id: treeSitterLanguage, + parser: Parser(), + supportsInjections: false, + ranges: [range.range]) + + if layerSet.contains(layer) { + // If we've found this layer, it means it should exist after an edit. + touchedLayers.remove(layer) + } else { + // New range, make a new layer! + if let addedLayer = addLanguageLayer(layerId: treeSitterLanguage, readBlock: readBlock) { + addedLayer.ranges = [range.range] + addedLayer.parser.includedRanges = addedLayer.ranges.map { $0.tsRange } + addedLayer.tree = createTree(parser: addedLayer.parser, readBlock: readBlock) + + layerSet.insert(addedLayer) + updatedRanges.insert(range: range.range) } } - - // Required for tree-sitter to work. Assumes no ranges are overlapping. - layer.ranges.sort() - } else { - // Add the language if not available - addLanguageLayer(layerId: treeSitterLanguage, readBlock: readBlock) - - let layerIndex = layers.count - 1 - guard layers.last?.id == treeSitterLanguage else { - continue - } - - layers[layerIndex].parser.includedRanges = ranges - .filter { $0.range.length > 0 } - .map { $0.tsRange } - .sorted() - layers[layerIndex].ranges = ranges.filter { $0.range.length > 0 }.map { $0.range } - layers[layerIndex].tree = createTree(parser: layers[layerIndex].parser, readBlock: readBlock) } } + return updatedRanges } } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift index db4e70184..392f5154b 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift @@ -10,14 +10,24 @@ import CodeEditLanguages import SwiftTreeSitter extension TreeSitterClient { - class LanguageLayer { + class LanguageLayer: Hashable { + /// Initialize a language layer + /// - Parameters: + /// - id: The ID of the layer. + /// - parser: A parser to use for the layer. + /// - supportsInjections: Set to true when the langauge supports the `injections` query. + /// - tree: The tree-sitter tree generated while editing/parsing a document. + /// - languageQuery: The language query used for fetching the associated `queries.scm` file + /// - ranges: All ranges this layer acts on. Must be kept in order and w/o overlap. init(id: TreeSitterLanguage, parser: Parser, + supportsInjections: Bool, tree: Tree? = nil, languageQuery: Query? = nil, ranges: [NSRange]) { self.id = id self.parser = parser + self.supportsInjections = supportsInjections self.tree = tree self.languageQuery = languageQuery self.ranges = ranges @@ -25,8 +35,18 @@ extension TreeSitterClient { let id: TreeSitterLanguage let parser: Parser + let supportsInjections: Bool var tree: Tree? var languageQuery: Query? var ranges: [NSRange] + + static func == (lhs: TreeSitterClient.LanguageLayer, rhs: TreeSitterClient.LanguageLayer) -> Bool { + return lhs.id == rhs.id && lhs.ranges == rhs.ranges + } + + func hash(into hasher: inout Hasher) { + hasher.combine(id) + hasher.combine(ranges) + } } } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index da4028a93..8bfa99720 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -29,10 +29,6 @@ final class TreeSitterClient: HighlightProviding { internal var textProvider: ResolvingQueryCursor.TextProvider - /// The queue to do tree-sitter work on for large edits/queries - internal let queue: DispatchQueue = DispatchQueue(label: "CodeEdit.CodeEditTextView.TreeSitter", - qos: .userInteractive) - /// Used to ensure safe use of the shared tree-sitter tree state in different sync/async contexts. internal let semaphore: DispatchSemaphore = DispatchSemaphore(value: 1) @@ -68,6 +64,7 @@ final class TreeSitterClient: HighlightProviding { layers = [ LanguageLayer(id: codeLanguage.id, parser: Parser(), + supportsInjections: codeLanguage.additionalHighlights?.contains("injections") ?? false, tree: nil, languageQuery: TreeSitterModel.shared.query(for: codeLanguage.id), ranges: []) @@ -87,10 +84,15 @@ final class TreeSitterClient: HighlightProviding { layers[0].tree = createTree(parser: layers[0].parser, readBlock: readBlock) + var layerSet = Set(arrayLiteral: layers[0]) + var touchedLayers = Set() + var idx = 0 while idx < layers.count { updateInjectedLanguageLayers(textView: textView, - language: layers[idx], + layer: layers[idx], + layerSet: &layerSet, + touchedLayers: &touchedLayers, readBlock: readBlock) idx += 1 @@ -115,29 +117,68 @@ final class TreeSitterClient: HighlightProviding { let readBlock = createReadBlock(textView: textView) var rangeSet = IndexSet() - // Loop through all layers and apply the edit, and query for any changed byte ranges. - // Using a while loop b/c `updateInjectedLanguageLayers` can append new layers during the loop. + // Helper data structure for finding existing layers in O(1) when adding injected layers + var layerSet = Set(minimumCapacity: layers.count) + // Tracks which layers were not touched at some point during the edit. Any layers left in this set + // after the second loop are removed. + var touchedLayers = Set(minimumCapacity: layers.count) + + // Loop through all layers, apply edits & find changed byte ranges. + for layerIdx in (0.. LanguageLayer? { + guard let language = CodeLanguage.allLanguages.first(where: { $0.id == layerId }), + let parserLanguage = language.language else { - return + return nil } - try? newLayer.parser.setLanguage(parserLanguage) + let newLayer = LanguageLayer( + id: layerId, + parser: Parser(), + supportsInjections: language.additionalHighlights?.contains("injections") ?? false, + tree: nil, + languageQuery: TreeSitterModel.shared.query(for: layerId), + ranges: [] + ) + + do { + try newLayer.parser.setLanguage(parserLanguage) + } catch { + return nil + } layers.append(newLayer) + return newLayer } /// Creates a tree-sitter tree. @@ -219,7 +267,7 @@ final class TreeSitterClient: HighlightProviding { let location = byteOffset / 2 let end = min(location + (1024), limit) if location > end { - assertionFailure("location is greater than end") + // Ignore and return nothing, tree-sitter's internal tree can be incorrect in some situations. return nil } let range = NSRange(location.. Date: Fri, 24 Mar 2023 16:04:33 -0500 Subject: [PATCH 08/10] Fix parameter list code style --- .../Extensions/NSRange+/NSRange+TSRange.swift | 6 ++- .../TreeSitter/TreeSitterClient+Edit.swift | 52 +++++++++++-------- .../TreeSitterClient+Highlight.swift | 8 +-- .../TreeSitterClient+LanguageLayer.swift | 14 ++--- 4 files changed, 48 insertions(+), 32 deletions(-) diff --git a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+TSRange.swift b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+TSRange.swift index 319763999..a0273784e 100644 --- a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+TSRange.swift +++ b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+TSRange.swift @@ -10,7 +10,9 @@ import SwiftTreeSitter extension NSRange { var tsRange: TSRange { - return TSRange(points: .zero..<(.zero), - bytes: (UInt32(self.location) * 2)..<(UInt32(self.location + self.length) * 2)) + return TSRange( + points: .zero..<(.zero), + bytes: (UInt32(self.location) * 2)..<(UInt32(self.location + self.length) * 2) + ) } } diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift index fa706c594..b7a31329e 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Edit.swift @@ -18,14 +18,18 @@ extension TreeSitterClient { /// - language: The language to use. /// - readBlock: A callback for fetching blocks of text. /// - Returns: An array of distinct `NSRanges` that need to be re-highlighted. - func findChangedByteRanges(textView: HighlighterTextView, - edit: InputEdit, - layer: LanguageLayer, - readBlock: @escaping Parser.ReadBlock) -> [NSRange] { - let (oldTree, newTree) = calculateNewState(tree: layer.tree, - parser: layer.parser, - edit: edit, - readBlock: readBlock) + func findChangedByteRanges( + textView: HighlighterTextView, + edit: InputEdit, + layer: LanguageLayer, + readBlock: @escaping Parser.ReadBlock + ) -> [NSRange] { + let (oldTree, newTree) = calculateNewState( + tree: layer.tree, + parser: layer.parser, + edit: edit, + readBlock: readBlock + ) if oldTree == nil && newTree == nil { // There was no existing tree, make a new one and return all indexes. layer.tree = createTree(parser: layer.parser, readBlock: readBlock) @@ -47,10 +51,12 @@ extension TreeSitterClient { /// - edit: The edit to apply. /// - readBlock: The block to use to read text. /// - Returns: (The old state, the new state). - internal func calculateNewState(tree: Tree?, - parser: Parser, - edit: InputEdit, - readBlock: @escaping Parser.ReadBlock) -> (Tree?, Tree?) { + internal func calculateNewState( + tree: Tree?, + parser: Parser, + edit: InputEdit, + readBlock: @escaping Parser.ReadBlock + ) -> (Tree?, Tree?) { guard let oldTree = tree else { return (nil, nil) } @@ -96,11 +102,13 @@ extension TreeSitterClient { /// - readBlock: A completion block for reading from text storage efficiently. /// - Returns: An index set of any updated indexes. @discardableResult - internal func updateInjectedLanguageLayers(textView: HighlighterTextView, - layer: LanguageLayer, - layerSet: inout Set, - touchedLayers: inout Set, - readBlock: @escaping Parser.ReadBlock) -> IndexSet { + internal func updateInjectedLanguageLayers( + textView: HighlighterTextView, + layer: LanguageLayer, + layerSet: inout Set, + touchedLayers: inout Set, + readBlock: @escaping Parser.ReadBlock + ) -> IndexSet { guard let tree = layer.tree, let rootNode = tree.rootNode, let cursor = layer.languageQuery?.execute(node: rootNode, in: tree) else { @@ -126,10 +134,12 @@ extension TreeSitterClient { for range in ranges { // Temp layer object for - let layer = LanguageLayer(id: treeSitterLanguage, - parser: Parser(), - supportsInjections: false, - ranges: [range.range]) + let layer = LanguageLayer( + id: treeSitterLanguage, + parser: Parser(), + supportsInjections: false, + ranges: [range.range] + ) if layerSet.contains(layer) { // If we've found this layer, it means it should exist after an edit. diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift index 8b1198029..de8764f0a 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+Highlight.swift @@ -17,9 +17,11 @@ extension TreeSitterClient { /// - textView: A text view to use for contextual data. /// - range: The range to query for. /// - Returns: Any ranges to highlight. - internal func queryLayerHighlights(layer: LanguageLayer, - textView: HighlighterTextView, - range: NSRange) -> [HighlightRange] { + internal func queryLayerHighlights( + layer: LanguageLayer, + textView: HighlighterTextView, + range: NSRange + ) -> [HighlightRange] { // Make sure we don't change the tree while we copy it. self.semaphore.wait() diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift index 392f5154b..c5ab2cc15 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient+LanguageLayer.swift @@ -19,12 +19,14 @@ extension TreeSitterClient { /// - tree: The tree-sitter tree generated while editing/parsing a document. /// - languageQuery: The language query used for fetching the associated `queries.scm` file /// - ranges: All ranges this layer acts on. Must be kept in order and w/o overlap. - init(id: TreeSitterLanguage, - parser: Parser, - supportsInjections: Bool, - tree: Tree? = nil, - languageQuery: Query? = nil, - ranges: [NSRange]) { + init( + id: TreeSitterLanguage, + parser: Parser, + supportsInjections: Bool, + tree: Tree? = nil, + languageQuery: Query? = nil, + ranges: [NSRange] + ) { self.id = id self.parser = parser self.supportsInjections = supportsInjections From 7c038175fa2d9fccebc9c464c94e594e0b25380c Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Fri, 24 Mar 2023 16:19:15 -0500 Subject: [PATCH 09/10] Fix lint errors --- .../TreeSitter/TreeSitterClient.swift | 105 ++++++++++-------- 1 file changed, 59 insertions(+), 46 deletions(-) diff --git a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift index 8bfa99720..66276576f 100644 --- a/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift +++ b/Sources/CodeEditTextView/TreeSitter/TreeSitterClient.swift @@ -62,12 +62,14 @@ final class TreeSitterClient: HighlightProviding { primaryLayer = codeLanguage.id layers = [ - LanguageLayer(id: codeLanguage.id, - parser: Parser(), - supportsInjections: codeLanguage.additionalHighlights?.contains("injections") ?? false, - tree: nil, - languageQuery: TreeSitterModel.shared.query(for: codeLanguage.id), - ranges: []) + LanguageLayer( + id: codeLanguage.id, + parser: Parser(), + supportsInjections: codeLanguage.additionalHighlights?.contains("injections") ?? false, + tree: nil, + languageQuery: TreeSitterModel.shared.query(for: codeLanguage.id), + ranges: [] + ) ] if let treeSitterLanguage = codeLanguage.language { @@ -81,19 +83,23 @@ final class TreeSitterClient: HighlightProviding { func setUp(textView: HighlighterTextView) { let readBlock = createReadBlock(textView: textView) - layers[0].tree = createTree(parser: layers[0].parser, - readBlock: readBlock) + layers[0].tree = createTree( + parser: layers[0].parser, + readBlock: readBlock + ) var layerSet = Set(arrayLiteral: layers[0]) var touchedLayers = Set() var idx = 0 while idx < layers.count { - updateInjectedLanguageLayers(textView: textView, - layer: layers[idx], - layerSet: &layerSet, - touchedLayers: &touchedLayers, - readBlock: readBlock) + updateInjectedLanguageLayers( + textView: textView, + layer: layers[idx], + layerSet: &layerSet, + touchedLayers: &touchedLayers, + readBlock: readBlock + ) idx += 1 } @@ -106,14 +112,13 @@ final class TreeSitterClient: HighlightProviding { /// - range: The range of the edit. /// - delta: The length of the edit, can be negative for deletions. /// - completion: The function to call with an `IndexSet` containing all Indices to invalidate. - func applyEdit(textView: HighlighterTextView, - range: NSRange, - delta: Int, - completion: @escaping ((IndexSet) -> Void)) { - guard let edit = InputEdit(range: range, delta: delta, oldEndPoint: .zero) else { - return - } - + func applyEdit( + textView: HighlighterTextView, + range: NSRange, + delta: Int, + completion: @escaping ((IndexSet) -> Void) + ) { + guard let edit = InputEdit(range: range, delta: delta, oldEndPoint: .zero) else { return } let readBlock = createReadBlock(textView: textView) var rangeSet = IndexSet() @@ -146,10 +151,12 @@ final class TreeSitterClient: HighlightProviding { layer.parser.includedRanges = layer.ranges.map { $0.tsRange } rangeSet.insert( - ranges: findChangedByteRanges(textView: textView, - edit: edit, - layer: layer, - readBlock: readBlock) + ranges: findChangedByteRanges( + textView: textView, + edit: edit, + layer: layer, + readBlock: readBlock + ) ) layerSet.insert(layer) @@ -163,11 +170,13 @@ final class TreeSitterClient: HighlightProviding { if layer.supportsInjections { rangeSet.formUnion( - updateInjectedLanguageLayers(textView: textView, - layer: layer, - layerSet: &layerSet, - touchedLayers: &touchedLayers, - readBlock: readBlock) + updateInjectedLanguageLayers( + textView: textView, + layer: layer, + layerSet: &layerSet, + touchedLayers: &touchedLayers, + readBlock: readBlock + ) ) } @@ -175,9 +184,7 @@ final class TreeSitterClient: HighlightProviding { } // Delete any layers that weren't touched at some point during the edit. - for layerIdx in (0.. Void))) { + func queryHighlightsFor( + textView: HighlighterTextView, + range: NSRange, + completion: @escaping ((([HighlightRange]) -> Void)) + ) { var highlights: [HighlightRange] = [] var injectedSet = IndexSet(integersIn: range) @@ -197,11 +206,11 @@ final class TreeSitterClient: HighlightProviding { // Query injected only if a layer's ranges intersects with `range` for layerRange in layer.ranges { if let rangeIntersection = range.intersection(layerRange) { - highlights.append( - contentsOf: queryLayerHighlights(layer: layer, - textView: textView, - range: rangeIntersection) - ) + highlights.append(contentsOf: queryLayerHighlights( + layer: layer, + textView: textView, + range: rangeIntersection + )) injectedSet.remove(integersIn: rangeIntersection) } @@ -210,9 +219,11 @@ final class TreeSitterClient: HighlightProviding { // Query primary for any ranges that weren't used in the injected layers. for range in injectedSet.rangeView { - highlights.append(contentsOf: queryLayerHighlights(layer: layers[0], - textView: textView, - range: NSRange(range))) + highlights.append(contentsOf: queryLayerHighlights( + layer: layers[0], + textView: textView, + range: NSRange(range) + )) } completion(highlights) @@ -225,8 +236,10 @@ final class TreeSitterClient: HighlightProviding { /// - Parameters: /// - layerId: A language ID to add as a layer. /// - readBlock: Completion called for efficient string lookup. - internal func addLanguageLayer(layerId: TreeSitterLanguage, - readBlock: @escaping Parser.ReadBlock) -> LanguageLayer? { + internal func addLanguageLayer( + layerId: TreeSitterLanguage, + readBlock: @escaping Parser.ReadBlock + ) -> LanguageLayer? { guard let language = CodeLanguage.allLanguages.first(where: { $0.id == layerId }), let parserLanguage = language.language else { From 7d7c5a6c6050986ca159ed17324d7d93474036e7 Mon Sep 17 00:00:00 2001 From: Khan Winter <35942988+thecoolwinter@users.noreply.github.com> Date: Fri, 24 Mar 2023 16:21:04 -0500 Subject: [PATCH 10/10] Update NSRange+Comparable file header --- .../Extensions/NSRange+/NSRange+Comparable.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+Comparable.swift b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+Comparable.swift index 662287ef9..15c7e3b12 100644 --- a/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+Comparable.swift +++ b/Sources/CodeEditTextView/Extensions/NSRange+/NSRange+Comparable.swift @@ -1,5 +1,5 @@ // -// File.swift +// NSRange+Comparable.swift // // // Created by Khan Winter on 3/15/23.