From eedfee6864352e88a10cae57700263480a9978db Mon Sep 17 00:00:00 2001 From: Gustaf Carefall <106698658+Gustaf-C@users.noreply.github.com> Date: Thu, 2 Apr 2026 17:15:55 +0200 Subject: [PATCH 01/10] Basic support for semantic highlighting --- .../semanticTokens/SemanticTokensProvider.ts | 83 +++++++++++++++++++ src/server.ts | 18 +++- 2 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 src/providers/semanticTokens/SemanticTokensProvider.ts diff --git a/src/providers/semanticTokens/SemanticTokensProvider.ts b/src/providers/semanticTokens/SemanticTokensProvider.ts new file mode 100644 index 0000000..0282d2b --- /dev/null +++ b/src/providers/semanticTokens/SemanticTokensProvider.ts @@ -0,0 +1,83 @@ +import { SemanticTokens, SemanticTokensParams, TextDocuments, Range } from 'vscode-languageserver' +import { TextDocument } from 'vscode-languageserver-textdocument' +import FileInfoIndex, { MatlabFunctionScopeInfo, MatlabGlobalScopeInfo } from '../../indexing/FileInfoIndex' +import DocumentIndexer from '../../indexing/DocumentIndexer' + +interface VariableToken { + range: Range + typeIndex: number +} + +class SemanticTokensProvider { + constructor( + protected readonly documentIndexer: DocumentIndexer, + protected readonly fileInfoIndex: FileInfoIndex + ) { } + + async handleSemanticTokensRequest( + params: SemanticTokensParams, + documentManager: TextDocuments + ): Promise { + + const textDocument = documentManager.get(params.textDocument.uri) + if (!textDocument) return null + + await this.documentIndexer.ensureDocumentIndexIsUpdated(textDocument) + + const codeInfo = this.fileInfoIndex.codeInfoCache.get(params.textDocument.uri) + if (!codeInfo) { + return { data: [] } + } + + const tokens: VariableToken[] = [] + this.collectVariableTokens(codeInfo.globalScopeInfo, tokens) + + tokens.sort((a, b) => + (a.range.start.line - b.range.start.line) || + (a.range.start.character - b.range.start.character) + ) + + const data: number[] = [] + let prevLine = 0 + let prevStart = 0 + + for (const token of tokens) { + const line = token.range.start.line + const start = token.range.start.character + const length = token.range.end.character - token.range.start.character + + const deltaLine = line - prevLine + const deltaStart = deltaLine === 0 ? start - prevStart : start + + data.push(deltaLine, deltaStart, length, token.typeIndex, 0) + prevLine = line + prevStart = start + } + + return { data } + } + + private collectVariableTokens( + scope: MatlabGlobalScopeInfo | MatlabFunctionScopeInfo, + tokens: VariableToken[] + ): void { + for (const variableInfo of scope.variables.values()) { + for (const ref of variableInfo.references) { + if (ref.components.length > 0) { + const typeIndex = 0 + tokens.push({ range: ref.components[0].range, typeIndex }) + } + } + } + + for (const nestedFunc of scope.functionScopes.values()) { + if (nestedFunc.functionScopeInfo) { + this.collectVariableTokens(nestedFunc.functionScopeInfo, tokens) + } + } + } +} + +export const SEMANTIC_TOKEN_TYPES = ['variable'] +export const SEMANTIC_TOKEN_MODIFIERS: string[] = [] +export default SemanticTokensProvider \ No newline at end of file diff --git a/src/server.ts b/src/server.ts index 2e985ca..5486821 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,7 +1,7 @@ // Copyright 2022 - 2025 The MathWorks, Inc. import { TextDocument } from 'vscode-languageserver-textdocument' -import { ClientCapabilities, InitializeParams, InitializeResult, TextDocuments } from 'vscode-languageserver/node' +import { ClientCapabilities, InitializeParams, InitializeResult, TextDocuments, SemanticTokensRequest, SemanticTokensParams } from 'vscode-languageserver/node' import DocumentIndexer from './indexing/DocumentIndexer' import WorkspaceIndexer from './indexing/WorkspaceIndexer' import ConfigurationManager, { ConnectionTiming } from './lifecycle/ConfigurationManager' @@ -22,6 +22,7 @@ import PathResolver from './providers/navigation/PathResolver' import Indexer from './indexing/Indexer' import RenameSymbolProvider from './providers/rename/RenameSymbolProvider' import HighlightSymbolProvider from './providers/highlighting/HighlightSymbolProvider' +import SemanticTokensProvider, { SEMANTIC_TOKEN_TYPES, SEMANTIC_TOKEN_MODIFIERS } from './providers/semanticTokens/SemanticTokensProvider' import { RequestType } from './indexing/SymbolSearchService' import { cacheAndClearProxyEnvironmentVariables } from './utils/ProxyUtils' import MatlabDebugAdaptorServer from './debug/MatlabDebugAdaptorServer' @@ -73,6 +74,7 @@ export async function startServer (): Promise { const navigationSupportProvider = new NavigationSupportProvider(matlabLifecycleManager, fileInfoIndex, indexer, documentIndexer, pathResolver) const renameSymbolProvider = new RenameSymbolProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex) const highlightSymbolProvider = new HighlightSymbolProvider(matlabLifecycleManager, documentIndexer, indexer, fileInfoIndex) + const semanticTokensProvider = new SemanticTokensProvider(documentIndexer, fileInfoIndex) let pathSynchronizer: PathSynchronizer | null @@ -142,7 +144,14 @@ export async function startServer (): Promise { renameProvider: { prepareProvider: true }, - documentHighlightProvider: true + documentHighlightProvider: true, + semanticTokensProvider: { + legend: { + tokenTypes: SEMANTIC_TOKEN_TYPES, + tokenModifiers: SEMANTIC_TOKEN_MODIFIERS + }, + full: true + } } } @@ -361,6 +370,11 @@ export async function startServer (): Promise { connection.onDocumentHighlight(async params => { return await highlightSymbolProvider.handleDocumentHighlightRequest(params, documentManager) }) + + /** -------------- SEMANTIC TOKENS SUPPORT --------------- **/ + connection.onRequest(SemanticTokensRequest.method, async (params: SemanticTokensParams) => { + return await semanticTokensProvider.handleSemanticTokensRequest(params, documentManager) + }) } /** -------------------- Helper Functions -------------------- **/ From be7d2064d0415b0264f295bcb4c5ef06481d0f4b Mon Sep 17 00:00:00 2001 From: Gustaf Carefall <106698658+Gustaf-C@users.noreply.github.com> Date: Thu, 2 Apr 2026 22:34:15 +0200 Subject: [PATCH 02/10] Variable highlighting inside classes --- .../semanticTokens/SemanticTokensProvider.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/providers/semanticTokens/SemanticTokensProvider.ts b/src/providers/semanticTokens/SemanticTokensProvider.ts index 0282d2b..074fbe1 100644 --- a/src/providers/semanticTokens/SemanticTokensProvider.ts +++ b/src/providers/semanticTokens/SemanticTokensProvider.ts @@ -61,6 +61,8 @@ class SemanticTokensProvider { scope: MatlabGlobalScopeInfo | MatlabFunctionScopeInfo, tokens: VariableToken[] ): void { + + // Global scope, e.g. for scripts for (const variableInfo of scope.variables.values()) { for (const ref of variableInfo.references) { if (ref.components.length > 0) { @@ -70,6 +72,17 @@ class SemanticTokensProvider { } } + // Class scope, for class definitions and methods + const classScope = (scope as MatlabGlobalScopeInfo).classScope; + if (classScope) { + for (const nestedFunc of classScope.functionScopes.values()) { + if (nestedFunc.functionScopeInfo) { + this.collectVariableTokens(nestedFunc.functionScopeInfo, tokens); + } + } + } + + // Function scopes, for nested functions for (const nestedFunc of scope.functionScopes.values()) { if (nestedFunc.functionScopeInfo) { this.collectVariableTokens(nestedFunc.functionScopeInfo, tokens) From f68def6a0f9df320265406b253aecfa89f823b91 Mon Sep 17 00:00:00 2001 From: Gustaf Carefall <106698658+Gustaf-C@users.noreply.github.com> Date: Thu, 16 Apr 2026 19:16:35 +0200 Subject: [PATCH 03/10] Cleanup, fix highlighting for functions without arguments --- .../semanticTokens/SemanticTokensProvider.ts | 38 ++++++++++++++----- src/server.ts | 2 +- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/src/providers/semanticTokens/SemanticTokensProvider.ts b/src/providers/semanticTokens/SemanticTokensProvider.ts index 074fbe1..f22a922 100644 --- a/src/providers/semanticTokens/SemanticTokensProvider.ts +++ b/src/providers/semanticTokens/SemanticTokensProvider.ts @@ -1,4 +1,5 @@ import { SemanticTokens, SemanticTokensParams, TextDocuments, Range } from 'vscode-languageserver' +import MatlabLifecycleManager from '../../lifecycle/MatlabLifecycleManager' import { TextDocument } from 'vscode-languageserver-textdocument' import FileInfoIndex, { MatlabFunctionScopeInfo, MatlabGlobalScopeInfo } from '../../indexing/FileInfoIndex' import DocumentIndexer from '../../indexing/DocumentIndexer' @@ -10,6 +11,7 @@ interface VariableToken { class SemanticTokensProvider { constructor( + protected readonly matlabLifecycleManager: MatlabLifecycleManager, protected readonly documentIndexer: DocumentIndexer, protected readonly fileInfoIndex: FileInfoIndex ) { } @@ -19,6 +21,13 @@ class SemanticTokensProvider { documentManager: TextDocuments ): Promise { + // This request will be called constantly, should not connect to MATLAB just because it was called + const matlabConnection = await this.matlabLifecycleManager.getMatlabConnection(false) + if (matlabConnection == null) { + // If MATLAB is not connected, fall back to textmate + return null + } + const textDocument = documentManager.get(params.textDocument.uri) if (!textDocument) return null @@ -41,6 +50,7 @@ class SemanticTokensProvider { let prevLine = 0 let prevStart = 0 + // Encode semantic tokens using relative line and character positions for (const token of tokens) { const line = token.range.start.line const start = token.range.start.character @@ -62,17 +72,25 @@ class SemanticTokensProvider { tokens: VariableToken[] ): void { - // Global scope, e.g. for scripts - for (const variableInfo of scope.variables.values()) { - for (const ref of variableInfo.references) { - if (ref.components.length > 0) { - const typeIndex = 0 - tokens.push({ range: ref.components[0].range, typeIndex }) + // Variables: highlight only the first component as variable + for (const item of scope.variables.values()) { + for (const ref of item.references) { + if (ref.components[0]) { + tokens.push({ range: ref.components[0].range, typeIndex: 1 }) // variable + } + } + } + + // Functions/unbound: highlight only the first component as function + for (const item of scope.functionOrUnboundReferences.values()) { + for (const ref of item.references) { + if (ref.components[0]) { + tokens.push({ range: ref.components[0].range, typeIndex: 0 }) // function } } } - // Class scope, for class definitions and methods + // Class scope const classScope = (scope as MatlabGlobalScopeInfo).classScope; if (classScope) { for (const nestedFunc of classScope.functionScopes.values()) { @@ -82,7 +100,7 @@ class SemanticTokensProvider { } } - // Function scopes, for nested functions + // Function scopes for (const nestedFunc of scope.functionScopes.values()) { if (nestedFunc.functionScopeInfo) { this.collectVariableTokens(nestedFunc.functionScopeInfo, tokens) @@ -91,6 +109,6 @@ class SemanticTokensProvider { } } -export const SEMANTIC_TOKEN_TYPES = ['variable'] +export const SEMANTIC_TOKEN_TYPES = ['function', 'variable'] export const SEMANTIC_TOKEN_MODIFIERS: string[] = [] -export default SemanticTokensProvider \ No newline at end of file +export default SemanticTokensProvider diff --git a/src/server.ts b/src/server.ts index 5486821..37de51c 100644 --- a/src/server.ts +++ b/src/server.ts @@ -74,7 +74,7 @@ export async function startServer (): Promise { const navigationSupportProvider = new NavigationSupportProvider(matlabLifecycleManager, fileInfoIndex, indexer, documentIndexer, pathResolver) const renameSymbolProvider = new RenameSymbolProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex) const highlightSymbolProvider = new HighlightSymbolProvider(matlabLifecycleManager, documentIndexer, indexer, fileInfoIndex) - const semanticTokensProvider = new SemanticTokensProvider(documentIndexer, fileInfoIndex) + const semanticTokensProvider = new SemanticTokensProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex) let pathSynchronizer: PathSynchronizer | null From 5f65ad4069fd2294f46e2845f102f1b6e8ea4987 Mon Sep 17 00:00:00 2001 From: Gustaf Carefall <106698658+Gustaf-C@users.noreply.github.com> Date: Thu, 16 Apr 2026 20:48:43 +0200 Subject: [PATCH 04/10] Linting --- .../semanticTokens/SemanticTokensProvider.ts | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/src/providers/semanticTokens/SemanticTokensProvider.ts b/src/providers/semanticTokens/SemanticTokensProvider.ts index f22a922..373306f 100644 --- a/src/providers/semanticTokens/SemanticTokensProvider.ts +++ b/src/providers/semanticTokens/SemanticTokensProvider.ts @@ -10,17 +10,16 @@ interface VariableToken { } class SemanticTokensProvider { - constructor( + constructor ( protected readonly matlabLifecycleManager: MatlabLifecycleManager, protected readonly documentIndexer: DocumentIndexer, protected readonly fileInfoIndex: FileInfoIndex ) { } - async handleSemanticTokensRequest( + async handleSemanticTokensRequest ( params: SemanticTokensParams, documentManager: TextDocuments ): Promise { - // This request will be called constantly, should not connect to MATLAB just because it was called const matlabConnection = await this.matlabLifecycleManager.getMatlabConnection(false) if (matlabConnection == null) { @@ -29,22 +28,24 @@ class SemanticTokensProvider { } const textDocument = documentManager.get(params.textDocument.uri) - if (!textDocument) return null + if (textDocument == null) return null await this.documentIndexer.ensureDocumentIndexIsUpdated(textDocument) const codeInfo = this.fileInfoIndex.codeInfoCache.get(params.textDocument.uri) - if (!codeInfo) { + if (codeInfo == null) { return { data: [] } } const tokens: VariableToken[] = [] this.collectVariableTokens(codeInfo.globalScopeInfo, tokens) - tokens.sort((a, b) => - (a.range.start.line - b.range.start.line) || - (a.range.start.character - b.range.start.character) - ) + tokens.sort((a, b) => { + const lineDiff = a.range.start.line - b.range.start.line + if (lineDiff !== 0) return lineDiff + + return a.range.start.character - b.range.start.character + }) const data: number[] = [] let prevLine = 0 @@ -67,34 +68,29 @@ class SemanticTokensProvider { return { data } } - private collectVariableTokens( + private collectVariableTokens ( scope: MatlabGlobalScopeInfo | MatlabFunctionScopeInfo, tokens: VariableToken[] ): void { - // Variables: highlight only the first component as variable for (const item of scope.variables.values()) { for (const ref of item.references) { - if (ref.components[0]) { - tokens.push({ range: ref.components[0].range, typeIndex: 1 }) // variable - } + tokens.push({ range: ref.components[0].range, typeIndex: 1 }) // variable } } // Functions/unbound: highlight only the first component as function for (const item of scope.functionOrUnboundReferences.values()) { for (const ref of item.references) { - if (ref.components[0]) { - tokens.push({ range: ref.components[0].range, typeIndex: 0 }) // function - } + tokens.push({ range: ref.components[0].range, typeIndex: 0 }) // function } } // Class scope const classScope = (scope as MatlabGlobalScopeInfo).classScope; - if (classScope) { + if (classScope != null) { for (const nestedFunc of classScope.functionScopes.values()) { - if (nestedFunc.functionScopeInfo) { + if (nestedFunc.functionScopeInfo != null) { this.collectVariableTokens(nestedFunc.functionScopeInfo, tokens); } } @@ -102,7 +98,7 @@ class SemanticTokensProvider { // Function scopes for (const nestedFunc of scope.functionScopes.values()) { - if (nestedFunc.functionScopeInfo) { + if (nestedFunc.functionScopeInfo != null) { this.collectVariableTokens(nestedFunc.functionScopeInfo, tokens) } } From a09cc0d355965fb380d1191cbcd993b5077ff30b Mon Sep 17 00:00:00 2001 From: Gustaf Carefall <106698658+Gustaf-C@users.noreply.github.com> Date: Thu, 16 Apr 2026 22:03:43 +0200 Subject: [PATCH 05/10] Ensure highlighting is correct after opening the editor --- src/indexing/DocumentIndexer.ts | 8 ++++++++ src/server.ts | 16 +++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/indexing/DocumentIndexer.ts b/src/indexing/DocumentIndexer.ts index adb75f7..e6f577b 100644 --- a/src/indexing/DocumentIndexer.ts +++ b/src/indexing/DocumentIndexer.ts @@ -12,6 +12,7 @@ const INDEXING_DELAY = 500 // Delay (in ms) after keystroke before attempting to */ export default class DocumentIndexer { private readonly pendingFilesToIndex = new Map() + private onIndexed?: (uri: string) => void constructor ( private readonly indexer: Indexer, @@ -42,6 +43,7 @@ export default class DocumentIndexer { */ indexDocument (textDocument: TextDocument): void { void this.indexer.indexDocument(textDocument) + this.onIndexed?.(textDocument.uri) } /** @@ -73,5 +75,11 @@ export default class DocumentIndexer { if (!this.fileInfoIndex.codeInfoCache.has(uri)) { await this.indexer.indexDocument(textDocument) } + + this.onIndexed?.(uri) + } + + setOnIndexed (callback: (uri: string) => void): void { + this.onIndexed = callback } } diff --git a/src/server.ts b/src/server.ts index 37de51c..9b41008 100644 --- a/src/server.ts +++ b/src/server.ts @@ -273,7 +273,7 @@ export async function startServer (): Promise { reportFileOpened(params.document) void lintingSupportProvider.lintDocument(params.document) void documentIndexer.indexDocument(params.document) - + void navigationSupportProvider.handleDocumentSymbol(params.document.uri, documentManager, RequestType.DocumentSymbol) }) @@ -375,6 +375,20 @@ export async function startServer (): Promise { connection.onRequest(SemanticTokensRequest.method, async (params: SemanticTokensParams) => { return await semanticTokensProvider.handleSemanticTokensRequest(params, documentManager) }) + + documentIndexer.setOnIndexed(() => { + scheduleSemanticRefresh() + }) + + let refreshTimer: NodeJS.Timeout | undefined + + function scheduleSemanticRefresh (): void { + if (refreshTimer != null) clearTimeout(refreshTimer) + + refreshTimer = setTimeout(() => { + void connection.sendRequest('workspace/semanticTokens/refresh') + }, 150) + } } /** -------------------- Helper Functions -------------------- **/ From 0c868a9781ad59a37ac0b59377c53b8d05de73fa Mon Sep 17 00:00:00 2001 From: Gustaf Carefall <106698658+Gustaf-C@users.noreply.github.com> Date: Thu, 16 Apr 2026 22:09:38 +0200 Subject: [PATCH 06/10] Added comments --- src/providers/semanticTokens/SemanticTokensProvider.ts | 2 ++ src/server.ts | 3 +++ 2 files changed, 5 insertions(+) diff --git a/src/providers/semanticTokens/SemanticTokensProvider.ts b/src/providers/semanticTokens/SemanticTokensProvider.ts index 373306f..5e9b14c 100644 --- a/src/providers/semanticTokens/SemanticTokensProvider.ts +++ b/src/providers/semanticTokens/SemanticTokensProvider.ts @@ -40,6 +40,8 @@ class SemanticTokensProvider { const tokens: VariableToken[] = [] this.collectVariableTokens(codeInfo.globalScopeInfo, tokens) + // Sort tokens by their position in the document (line and character) + // This is necessary to encode them using relative positions tokens.sort((a, b) => { const lineDiff = a.range.start.line - b.range.start.line if (lineDiff !== 0) return lineDiff diff --git a/src/server.ts b/src/server.ts index 9b41008..c7241fa 100644 --- a/src/server.ts +++ b/src/server.ts @@ -376,6 +376,8 @@ export async function startServer (): Promise { return await semanticTokensProvider.handleSemanticTokensRequest(params, documentManager) }) + // Ensures that semantic tokens are refreshed after indexing, + // so highlighting is updated after opening the editor. documentIndexer.setOnIndexed(() => { scheduleSemanticRefresh() }) @@ -385,6 +387,7 @@ export async function startServer (): Promise { function scheduleSemanticRefresh (): void { if (refreshTimer != null) clearTimeout(refreshTimer) + // Delay sending the refresh notification to batch multiple indexing updates together refreshTimer = setTimeout(() => { void connection.sendRequest('workspace/semanticTokens/refresh') }, 150) From 59f1071c2dac7e0fa59f569bfb256b2bc0467770 Mon Sep 17 00:00:00 2001 From: Gustaf Carefall <106698658+Gustaf-C@users.noreply.github.com> Date: Fri, 17 Apr 2026 23:49:44 +0200 Subject: [PATCH 07/10] Code cleanup --- .../semanticTokens/SemanticTokensProvider.ts | 32 ++++++++++--------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/src/providers/semanticTokens/SemanticTokensProvider.ts b/src/providers/semanticTokens/SemanticTokensProvider.ts index 5e9b14c..492cd75 100644 --- a/src/providers/semanticTokens/SemanticTokensProvider.ts +++ b/src/providers/semanticTokens/SemanticTokensProvider.ts @@ -4,7 +4,7 @@ import { TextDocument } from 'vscode-languageserver-textdocument' import FileInfoIndex, { MatlabFunctionScopeInfo, MatlabGlobalScopeInfo } from '../../indexing/FileInfoIndex' import DocumentIndexer from '../../indexing/DocumentIndexer' -interface VariableToken { +interface SemanticToken { range: Range typeIndex: number } @@ -20,12 +20,10 @@ class SemanticTokensProvider { params: SemanticTokensParams, documentManager: TextDocuments ): Promise { - // This request will be called constantly, should not connect to MATLAB just because it was called + // This provider will be called constantly, should not connect to MATLAB just because it was called const matlabConnection = await this.matlabLifecycleManager.getMatlabConnection(false) - if (matlabConnection == null) { - // If MATLAB is not connected, fall back to textmate - return null - } + // If MATLAB is not connected, fall back to default highlighting + if (matlabConnection == null) return null const textDocument = documentManager.get(params.textDocument.uri) if (textDocument == null) return null @@ -33,12 +31,10 @@ class SemanticTokensProvider { await this.documentIndexer.ensureDocumentIndexIsUpdated(textDocument) const codeInfo = this.fileInfoIndex.codeInfoCache.get(params.textDocument.uri) - if (codeInfo == null) { - return { data: [] } - } + if (codeInfo == null) return null - const tokens: VariableToken[] = [] - this.collectVariableTokens(codeInfo.globalScopeInfo, tokens) + const tokens: SemanticToken[] = [] + this.collectSemanticTokens(codeInfo.globalScopeInfo, tokens) // Sort tokens by their position in the document (line and character) // This is necessary to encode them using relative positions @@ -70,9 +66,15 @@ class SemanticTokensProvider { return { data } } - private collectVariableTokens ( + /** + * Recursively collects semantic tokens for a given scope and its nested scopes. + * Tokens are appended to 'tokens' in-place. + * @param scope The scope from which semantic tokens should be collected + * @param tokens The array to which collected semantic tokens are appended + */ + private collectSemanticTokens ( scope: MatlabGlobalScopeInfo | MatlabFunctionScopeInfo, - tokens: VariableToken[] + tokens: SemanticToken[] ): void { // Variables: highlight only the first component as variable for (const item of scope.variables.values()) { @@ -93,7 +95,7 @@ class SemanticTokensProvider { if (classScope != null) { for (const nestedFunc of classScope.functionScopes.values()) { if (nestedFunc.functionScopeInfo != null) { - this.collectVariableTokens(nestedFunc.functionScopeInfo, tokens); + this.collectSemanticTokens(nestedFunc.functionScopeInfo, tokens); } } } @@ -101,7 +103,7 @@ class SemanticTokensProvider { // Function scopes for (const nestedFunc of scope.functionScopes.values()) { if (nestedFunc.functionScopeInfo != null) { - this.collectVariableTokens(nestedFunc.functionScopeInfo, tokens) + this.collectSemanticTokens(nestedFunc.functionScopeInfo, tokens) } } } From 982ea899e4aa45d1f1b424442d34a2ed62ae7bb4 Mon Sep 17 00:00:00 2001 From: Gustaf Carefall <106698658+Gustaf-C@users.noreply.github.com> Date: Sat, 18 Apr 2026 00:58:26 +0200 Subject: [PATCH 08/10] Move token refresh logic to a separate function outside the server --- .../setupSemanticTokenRefresh.ts | 25 +++++++++++++++++++ src/server.ts | 19 ++------------ 2 files changed, 27 insertions(+), 17 deletions(-) create mode 100644 src/providers/semanticTokens/setupSemanticTokenRefresh.ts diff --git a/src/providers/semanticTokens/setupSemanticTokenRefresh.ts b/src/providers/semanticTokens/setupSemanticTokenRefresh.ts new file mode 100644 index 0000000..c1c6a41 --- /dev/null +++ b/src/providers/semanticTokens/setupSemanticTokenRefresh.ts @@ -0,0 +1,25 @@ +import { Connection } from 'vscode-languageserver' +import DocumentIndexer from '../../indexing/DocumentIndexer' + +/** + * Wires semantic token invalidation to document indexing. + * + * When indexing completes, this schedules a debounced refresh request + * so the client re-requests semantic tokens and updates highlighting. + */ +function setupSemanticTokenRefresh ( + connection: Connection, + documentIndexer: DocumentIndexer +): void { + let refreshTimer: NodeJS.Timeout | undefined + + documentIndexer.setOnIndexed(() => { + if (refreshTimer != null) clearTimeout(refreshTimer) + + refreshTimer = setTimeout(() => { + void connection.sendRequest('workspace/semanticTokens/refresh') + }, 150) + }) +} + +export default setupSemanticTokenRefresh diff --git a/src/server.ts b/src/server.ts index c7241fa..af54830 100644 --- a/src/server.ts +++ b/src/server.ts @@ -23,6 +23,7 @@ import Indexer from './indexing/Indexer' import RenameSymbolProvider from './providers/rename/RenameSymbolProvider' import HighlightSymbolProvider from './providers/highlighting/HighlightSymbolProvider' import SemanticTokensProvider, { SEMANTIC_TOKEN_TYPES, SEMANTIC_TOKEN_MODIFIERS } from './providers/semanticTokens/SemanticTokensProvider' +import setupSemanticTokenRefresh from './providers/semanticTokens/setupSemanticTokenRefresh' import { RequestType } from './indexing/SymbolSearchService' import { cacheAndClearProxyEnvironmentVariables } from './utils/ProxyUtils' import MatlabDebugAdaptorServer from './debug/MatlabDebugAdaptorServer' @@ -375,23 +376,7 @@ export async function startServer (): Promise { connection.onRequest(SemanticTokensRequest.method, async (params: SemanticTokensParams) => { return await semanticTokensProvider.handleSemanticTokensRequest(params, documentManager) }) - - // Ensures that semantic tokens are refreshed after indexing, - // so highlighting is updated after opening the editor. - documentIndexer.setOnIndexed(() => { - scheduleSemanticRefresh() - }) - - let refreshTimer: NodeJS.Timeout | undefined - - function scheduleSemanticRefresh (): void { - if (refreshTimer != null) clearTimeout(refreshTimer) - - // Delay sending the refresh notification to batch multiple indexing updates together - refreshTimer = setTimeout(() => { - void connection.sendRequest('workspace/semanticTokens/refresh') - }, 150) - } + setupSemanticTokenRefresh(connection, documentIndexer) } /** -------------------- Helper Functions -------------------- **/ From 1cd073637aa7acd9e95cb46ac571f0658709bd3a Mon Sep 17 00:00:00 2001 From: Gustaf Carefall <106698658+Gustaf-C@users.noreply.github.com> Date: Sat, 18 Apr 2026 21:15:13 +0200 Subject: [PATCH 09/10] Moved setupSemanticTokensRefresh to the same file as the provider class --- .../semanticTokens/SemanticTokensProvider.ts | 28 +++++++++++++++++-- .../setupSemanticTokenRefresh.ts | 25 ----------------- src/server.ts | 5 ++-- 3 files changed, 27 insertions(+), 31 deletions(-) delete mode 100644 src/providers/semanticTokens/setupSemanticTokenRefresh.ts diff --git a/src/providers/semanticTokens/SemanticTokensProvider.ts b/src/providers/semanticTokens/SemanticTokensProvider.ts index 492cd75..c189f8f 100644 --- a/src/providers/semanticTokens/SemanticTokensProvider.ts +++ b/src/providers/semanticTokens/SemanticTokensProvider.ts @@ -1,4 +1,4 @@ -import { SemanticTokens, SemanticTokensParams, TextDocuments, Range } from 'vscode-languageserver' +import { SemanticTokens, SemanticTokensParams, TextDocuments, Range, Connection } from 'vscode-languageserver' import MatlabLifecycleManager from '../../lifecycle/MatlabLifecycleManager' import { TextDocument } from 'vscode-languageserver-textdocument' import FileInfoIndex, { MatlabFunctionScopeInfo, MatlabGlobalScopeInfo } from '../../indexing/FileInfoIndex' @@ -9,6 +9,9 @@ interface SemanticToken { typeIndex: number } +/** + * Handles requests for semantic tokens for a document. + */ class SemanticTokensProvider { constructor ( protected readonly matlabLifecycleManager: MatlabLifecycleManager, @@ -28,8 +31,6 @@ class SemanticTokensProvider { const textDocument = documentManager.get(params.textDocument.uri) if (textDocument == null) return null - await this.documentIndexer.ensureDocumentIndexIsUpdated(textDocument) - const codeInfo = this.fileInfoIndex.codeInfoCache.get(params.textDocument.uri) if (codeInfo == null) return null @@ -112,3 +113,24 @@ class SemanticTokensProvider { export const SEMANTIC_TOKEN_TYPES = ['function', 'variable'] export const SEMANTIC_TOKEN_MODIFIERS: string[] = [] export default SemanticTokensProvider + +/** + * Wires semantic token invalidation to document indexing. + * + * When indexing completes, this schedules a debounced refresh request + * so the client re-requests semantic tokens and updates highlighting. + */ +export function setupSemanticTokensRefresh ( + connection: Connection, + documentIndexer: DocumentIndexer +): void { + let refreshTimer: NodeJS.Timeout | undefined + + documentIndexer.setOnIndexed(() => { + if (refreshTimer != null) clearTimeout(refreshTimer) + + refreshTimer = setTimeout(() => { + void connection.sendRequest('workspace/semanticTokens/refresh') + }, 150) + }) +} diff --git a/src/providers/semanticTokens/setupSemanticTokenRefresh.ts b/src/providers/semanticTokens/setupSemanticTokenRefresh.ts deleted file mode 100644 index c1c6a41..0000000 --- a/src/providers/semanticTokens/setupSemanticTokenRefresh.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { Connection } from 'vscode-languageserver' -import DocumentIndexer from '../../indexing/DocumentIndexer' - -/** - * Wires semantic token invalidation to document indexing. - * - * When indexing completes, this schedules a debounced refresh request - * so the client re-requests semantic tokens and updates highlighting. - */ -function setupSemanticTokenRefresh ( - connection: Connection, - documentIndexer: DocumentIndexer -): void { - let refreshTimer: NodeJS.Timeout | undefined - - documentIndexer.setOnIndexed(() => { - if (refreshTimer != null) clearTimeout(refreshTimer) - - refreshTimer = setTimeout(() => { - void connection.sendRequest('workspace/semanticTokens/refresh') - }, 150) - }) -} - -export default setupSemanticTokenRefresh diff --git a/src/server.ts b/src/server.ts index af54830..4f5207d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -22,8 +22,7 @@ import PathResolver from './providers/navigation/PathResolver' import Indexer from './indexing/Indexer' import RenameSymbolProvider from './providers/rename/RenameSymbolProvider' import HighlightSymbolProvider from './providers/highlighting/HighlightSymbolProvider' -import SemanticTokensProvider, { SEMANTIC_TOKEN_TYPES, SEMANTIC_TOKEN_MODIFIERS } from './providers/semanticTokens/SemanticTokensProvider' -import setupSemanticTokenRefresh from './providers/semanticTokens/setupSemanticTokenRefresh' +import SemanticTokensProvider, { SEMANTIC_TOKEN_TYPES, SEMANTIC_TOKEN_MODIFIERS, setupSemanticTokensRefresh } from './providers/semanticTokens/SemanticTokensProvider' import { RequestType } from './indexing/SymbolSearchService' import { cacheAndClearProxyEnvironmentVariables } from './utils/ProxyUtils' import MatlabDebugAdaptorServer from './debug/MatlabDebugAdaptorServer' @@ -376,7 +375,7 @@ export async function startServer (): Promise { connection.onRequest(SemanticTokensRequest.method, async (params: SemanticTokensParams) => { return await semanticTokensProvider.handleSemanticTokensRequest(params, documentManager) }) - setupSemanticTokenRefresh(connection, documentIndexer) + setupSemanticTokensRefresh(connection, documentIndexer) } /** -------------------- Helper Functions -------------------- **/ From 672e8942adc324334eb8d44bb3ee7281e4b688a8 Mon Sep 17 00:00:00 2001 From: Gustaf Carefall <106698658+Gustaf-C@users.noreply.github.com> Date: Sat, 18 Apr 2026 22:06:18 +0200 Subject: [PATCH 10/10] Added unit tests for semantic tokens --- .../SemanticTokensProvider.test.ts | 296 ++++++++++++++++++ 1 file changed, 296 insertions(+) create mode 100644 tests/providers/semanticTokens/SemanticTokensProvider.test.ts diff --git a/tests/providers/semanticTokens/SemanticTokensProvider.test.ts b/tests/providers/semanticTokens/SemanticTokensProvider.test.ts new file mode 100644 index 0000000..aa08fe8 --- /dev/null +++ b/tests/providers/semanticTokens/SemanticTokensProvider.test.ts @@ -0,0 +1,296 @@ +import assert from 'assert' +import sinon from 'sinon' +import quibble from 'quibble' + +import { TextDocuments, Range } from 'vscode-languageserver' +import { TextDocument } from 'vscode-languageserver-textdocument' + +import MatlabLifecycleManager from '../../../src/lifecycle/MatlabLifecycleManager' +import DocumentIndexer from '../../../src/indexing/DocumentIndexer' +import Indexer from '../../../src/indexing/Indexer' +import FileInfoIndex, { MatlabFunctionScopeInfo, MatlabGlobalScopeInfo } from '../../../src/indexing/FileInfoIndex' +import ClientConnection from '../../../src/ClientConnection' +import getMockConnection from '../../mocks/Connection.mock' +import getMockMvm from '../../mocks/Mvm.mock' +import { dynamicImport } from '../../TestUtils' + +describe('SemanticTokenProvider', () => { + const URI = 'file://test.m' + + let matlabLifecycleManager: MatlabLifecycleManager + let fileInfoIndex: FileInfoIndex + let indexer: Indexer + let documentIndexer: DocumentIndexer + + let semanticTokensProvider: any + let documentManager: TextDocuments + + const setup = () => { + const mockMvm = getMockMvm() + + matlabLifecycleManager = new MatlabLifecycleManager() + fileInfoIndex = new FileInfoIndex() + indexer = new Indexer(matlabLifecycleManager, mockMvm, fileInfoIndex) + documentIndexer = new DocumentIndexer(indexer, fileInfoIndex) + + documentManager = new TextDocuments(TextDocument) + + sinon.stub(matlabLifecycleManager, 'getMatlabConnection').resolves({} as any) + + type SemanticTokensProviderExports = typeof import('../../../src/providers/semanticTokens/SemanticTokensProvider') + const { default: SemanticTokensProvider } = dynamicImport( + module, '../../../src/providers/semanticTokens/SemanticTokensProvider') + + semanticTokensProvider = new SemanticTokensProvider( + matlabLifecycleManager, + documentIndexer, + fileInfoIndex + ) + + const doc = TextDocument.create( + URI, 'matlab', 1, 'abc' + ) + + sinon.stub(documentManager, 'get').returns(doc) + } + + const teardown = () => { + quibble.reset() + sinon.restore() + } + + before(() => { + ClientConnection._setConnection(getMockConnection()) + }) + + after(() => { + ClientConnection._clearConnection() + }) + + describe('#handleSemanticTokensRequest', () => { + beforeEach(() => setup()) + afterEach(() => teardown()) + + it('should return null if there is no MATLAB connection', async () => { + (matlabLifecycleManager.getMatlabConnection as sinon.SinonStub).resolves(null) + + const res = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri: URI } }, documentManager + ) + + assert.strictEqual(res, null, 'Result should be null when there is no MATLAB connection') + }) + + it('should return null if there is no document at the given URI', async () => { + (documentManager.get as sinon.SinonStub).returns(undefined) + + const res = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri: URI } }, documentManager + ) + + assert.strictEqual(res, null, 'Result should be null when there is no document at the given URI') + }) + + it('should return null if codeinfo is null', async () => { + fileInfoIndex.codeInfoCache.set(URI, null as any) + + const res = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri: URI } }, documentManager + ) + + assert.strictEqual(res, null, 'Result should be null when codeinfo is null') + }) + + it('should mark foo as a semantic function', async () => { + const uri = URI + + const range = { + start: { line: 0, character: 0 }, + end: { line: 0, character: 3 } // "foo" + } + + fileInfoIndex.codeInfoCache.set( + URI, + createCodeInfo({ + functions: [ + { name: 'foo', range: range } + ] + }) + ) + + const result = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri } }, + documentManager + ) + + assert.ok(result, 'Expected semantic tokens result') + + const [deltaLine, deltaStart, length, typeIndex] = result!.data + + assert.strictEqual(typeIndex, 0, 'Expected function token type') + assert.strictEqual(length, 3, 'Expected token length for "foo"') + }) + + it('should mark x as a semantic variable', async () => { + const uri = URI + + const range = { + start: { line: 0, character: 0 }, + end: { line: 0, character: 1 } + } + + fileInfoIndex.codeInfoCache.set( + URI, + createCodeInfo({ + variables: [ + { name: 'x', range: range } + ] + }) + ) + + const result = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri } }, + documentManager + ) + + assert.ok(result) + + const [deltaLine, deltaStart, length, typeIndex] = result!.data + + assert.strictEqual(typeIndex, 1) // variable + assert.strictEqual(length, 1) + }) + + it('should encode delta correctly for semantic tokens on same line', async () => { + const uri = URI + + const range1 = { + start: { line: 0, character: 1 }, + end: { line: 0, character: 2 } // "x" + } as unknown as Range + + const range2 = { + start: { line: 0, character: 5 }, + end: { line: 0, character: 6 } // "y" + } as unknown as Range + + fileInfoIndex.codeInfoCache.set( + URI, + createCodeInfo({ + variables: [ + { name: 'x', range: range1 }, + { name: 'y', range: range2 } + ] + }) + ) + + const result = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri } } as any, + documentManager + ) + + assert.ok(result, 'Expected semantic tokens result') + + // Token 1 + const [dl1, ds1, len1, type1] = result!.data.slice(0, 4) + + // Token 2 + const [dl2, ds2, len2, type2] = result!.data.slice(5, 9) + + // First token (absolute) + assert.strictEqual(dl1, 0) + assert.strictEqual(ds1, 1) + + // Second token: + // same line → deltaLine = 0 + assert.strictEqual(dl2, 0, 'Expected same line') + + // relative start: 5 - 1 = 4 + assert.strictEqual(ds2, 4, 'Expected relative deltaStart') + }) + + it('should encode delta correctly for semantic tokens on different lines', async () => { + const uri = URI + + const range1 = { + start: { line: 0, character: 0 }, + end: { line: 0, character: 1 } // "x" + } + + const range2 = { + start: { line: 2, character: 7 }, + end: { line: 2, character: 8 } // "y" + } + + fileInfoIndex.codeInfoCache.set( + URI, + createCodeInfo({ + variables: [ // Note: Variables are intentionally added in reverse order to test sorting of tokens + { name: 'y', range: range2 }, + { name: 'x', range: range1 } + ] + }) + ) + + const result = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri } } as any, + documentManager + ) + + assert.ok(result, 'Expected semantic tokens result') + + // Token 1 + const [dl1, ds1, len1, type1] = result!.data.slice(0, 4) + + // Token 2 + const [dl2, ds2, len2, type2] = result!.data.slice(5, 9) + + // First token: absolute position + assert.strictEqual(dl1, 0, 'Expected first token to have dl1 = 0, sorting should ensure this is the case') + assert.strictEqual(ds1, 0, 'Expected first token to have ds1 = 0, sorting should ensure this is the case') + + // Second token: + // line 2 - line 0 = 2 + assert.strictEqual(dl2, 2, 'Expected deltaLine = 2') + + // since new line → start should NOT be relative + assert.strictEqual(ds2, 7, 'Expected absolute start on new line') + }) + }) +}) + +function createCodeInfo({ + variables = [], + functions = [] +}: { + variables?: Array<{ name: string, range: Range }> + functions?: Array<{ name: string, range: Range }> +}) { + return { + globalScopeInfo: { + variables: new Map( + variables.map(v => [ + v.name, + { + references: [ + { components: [{ range: v.range }] } + ] + } + ]) + ), + + functionOrUnboundReferences: new Map( + functions.map(f => [ + f.name, + { + references: [ + { components: [{ range: f.range }] } + ] + } + ]) + ), + + functionScopes: new Map() + } + } as any +} \ No newline at end of file