diff --git a/src/indexing/DocumentIndexer.ts b/src/indexing/DocumentIndexer.ts index adb75f7..e6f577b 100644 --- a/src/indexing/DocumentIndexer.ts +++ b/src/indexing/DocumentIndexer.ts @@ -12,6 +12,7 @@ const INDEXING_DELAY = 500 // Delay (in ms) after keystroke before attempting to */ export default class DocumentIndexer { private readonly pendingFilesToIndex = new Map() + private onIndexed?: (uri: string) => void constructor ( private readonly indexer: Indexer, @@ -42,6 +43,7 @@ export default class DocumentIndexer { */ indexDocument (textDocument: TextDocument): void { void this.indexer.indexDocument(textDocument) + this.onIndexed?.(textDocument.uri) } /** @@ -73,5 +75,11 @@ export default class DocumentIndexer { if (!this.fileInfoIndex.codeInfoCache.has(uri)) { await this.indexer.indexDocument(textDocument) } + + this.onIndexed?.(uri) + } + + setOnIndexed (callback: (uri: string) => void): void { + this.onIndexed = callback } } diff --git a/src/providers/semanticTokens/SemanticTokensProvider.ts b/src/providers/semanticTokens/SemanticTokensProvider.ts new file mode 100644 index 0000000..c189f8f --- /dev/null +++ b/src/providers/semanticTokens/SemanticTokensProvider.ts @@ -0,0 +1,136 @@ +import { SemanticTokens, SemanticTokensParams, TextDocuments, Range, Connection } from 'vscode-languageserver' +import MatlabLifecycleManager from '../../lifecycle/MatlabLifecycleManager' +import { TextDocument } from 'vscode-languageserver-textdocument' +import FileInfoIndex, { MatlabFunctionScopeInfo, MatlabGlobalScopeInfo } from '../../indexing/FileInfoIndex' +import DocumentIndexer from '../../indexing/DocumentIndexer' + +interface SemanticToken { + range: Range + typeIndex: number +} + +/** + * Handles requests for semantic tokens for a document. + */ +class SemanticTokensProvider { + constructor ( + protected readonly matlabLifecycleManager: MatlabLifecycleManager, + protected readonly documentIndexer: DocumentIndexer, + protected readonly fileInfoIndex: FileInfoIndex + ) { } + + async handleSemanticTokensRequest ( + params: SemanticTokensParams, + documentManager: TextDocuments + ): Promise { + // This provider will be called constantly, should not connect to MATLAB just because it was called + const matlabConnection = await this.matlabLifecycleManager.getMatlabConnection(false) + // If MATLAB is not connected, fall back to default highlighting + if (matlabConnection == null) return null + + const textDocument = documentManager.get(params.textDocument.uri) + if (textDocument == null) return null + + const codeInfo = this.fileInfoIndex.codeInfoCache.get(params.textDocument.uri) + if (codeInfo == null) return null + + const tokens: SemanticToken[] = [] + this.collectSemanticTokens(codeInfo.globalScopeInfo, tokens) + + // Sort tokens by their position in the document (line and character) + // This is necessary to encode them using relative positions + tokens.sort((a, b) => { + const lineDiff = a.range.start.line - b.range.start.line + if (lineDiff !== 0) return lineDiff + + return a.range.start.character - b.range.start.character + }) + + const data: number[] = [] + let prevLine = 0 + let prevStart = 0 + + // Encode semantic tokens using relative line and character positions + for (const token of tokens) { + const line = token.range.start.line + const start = token.range.start.character + const length = token.range.end.character - token.range.start.character + + const deltaLine = line - prevLine + const deltaStart = deltaLine === 0 ? start - prevStart : start + + data.push(deltaLine, deltaStart, length, token.typeIndex, 0) + prevLine = line + prevStart = start + } + + return { data } + } + + /** + * Recursively collects semantic tokens for a given scope and its nested scopes. + * Tokens are appended to 'tokens' in-place. + * @param scope The scope from which semantic tokens should be collected + * @param tokens The array to which collected semantic tokens are appended + */ + private collectSemanticTokens ( + scope: MatlabGlobalScopeInfo | MatlabFunctionScopeInfo, + tokens: SemanticToken[] + ): void { + // Variables: highlight only the first component as variable + for (const item of scope.variables.values()) { + for (const ref of item.references) { + tokens.push({ range: ref.components[0].range, typeIndex: 1 }) // variable + } + } + + // Functions/unbound: highlight only the first component as function + for (const item of scope.functionOrUnboundReferences.values()) { + for (const ref of item.references) { + tokens.push({ range: ref.components[0].range, typeIndex: 0 }) // function + } + } + + // Class scope + const classScope = (scope as MatlabGlobalScopeInfo).classScope; + if (classScope != null) { + for (const nestedFunc of classScope.functionScopes.values()) { + if (nestedFunc.functionScopeInfo != null) { + this.collectSemanticTokens(nestedFunc.functionScopeInfo, tokens); + } + } + } + + // Function scopes + for (const nestedFunc of scope.functionScopes.values()) { + if (nestedFunc.functionScopeInfo != null) { + this.collectSemanticTokens(nestedFunc.functionScopeInfo, tokens) + } + } + } +} + +export const SEMANTIC_TOKEN_TYPES = ['function', 'variable'] +export const SEMANTIC_TOKEN_MODIFIERS: string[] = [] +export default SemanticTokensProvider + +/** + * Wires semantic token invalidation to document indexing. + * + * When indexing completes, this schedules a debounced refresh request + * so the client re-requests semantic tokens and updates highlighting. + */ +export function setupSemanticTokensRefresh ( + connection: Connection, + documentIndexer: DocumentIndexer +): void { + let refreshTimer: NodeJS.Timeout | undefined + + documentIndexer.setOnIndexed(() => { + if (refreshTimer != null) clearTimeout(refreshTimer) + + refreshTimer = setTimeout(() => { + void connection.sendRequest('workspace/semanticTokens/refresh') + }, 150) + }) +} diff --git a/src/server.ts b/src/server.ts index 2e985ca..4f5207d 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,7 +1,7 @@ // Copyright 2022 - 2025 The MathWorks, Inc. import { TextDocument } from 'vscode-languageserver-textdocument' -import { ClientCapabilities, InitializeParams, InitializeResult, TextDocuments } from 'vscode-languageserver/node' +import { ClientCapabilities, InitializeParams, InitializeResult, TextDocuments, SemanticTokensRequest, SemanticTokensParams } from 'vscode-languageserver/node' import DocumentIndexer from './indexing/DocumentIndexer' import WorkspaceIndexer from './indexing/WorkspaceIndexer' import ConfigurationManager, { ConnectionTiming } from './lifecycle/ConfigurationManager' @@ -22,6 +22,7 @@ import PathResolver from './providers/navigation/PathResolver' import Indexer from './indexing/Indexer' import RenameSymbolProvider from './providers/rename/RenameSymbolProvider' import HighlightSymbolProvider from './providers/highlighting/HighlightSymbolProvider' +import SemanticTokensProvider, { SEMANTIC_TOKEN_TYPES, SEMANTIC_TOKEN_MODIFIERS, setupSemanticTokensRefresh } from './providers/semanticTokens/SemanticTokensProvider' import { RequestType } from './indexing/SymbolSearchService' import { cacheAndClearProxyEnvironmentVariables } from './utils/ProxyUtils' import MatlabDebugAdaptorServer from './debug/MatlabDebugAdaptorServer' @@ -73,6 +74,7 @@ export async function startServer (): Promise { const navigationSupportProvider = new NavigationSupportProvider(matlabLifecycleManager, fileInfoIndex, indexer, documentIndexer, pathResolver) const renameSymbolProvider = new RenameSymbolProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex) const highlightSymbolProvider = new HighlightSymbolProvider(matlabLifecycleManager, documentIndexer, indexer, fileInfoIndex) + const semanticTokensProvider = new SemanticTokensProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex) let pathSynchronizer: PathSynchronizer | null @@ -142,7 +144,14 @@ export async function startServer (): Promise { renameProvider: { prepareProvider: true }, - documentHighlightProvider: true + documentHighlightProvider: true, + semanticTokensProvider: { + legend: { + tokenTypes: SEMANTIC_TOKEN_TYPES, + tokenModifiers: SEMANTIC_TOKEN_MODIFIERS + }, + full: true + } } } @@ -264,7 +273,7 @@ export async function startServer (): Promise { reportFileOpened(params.document) void lintingSupportProvider.lintDocument(params.document) void documentIndexer.indexDocument(params.document) - + void navigationSupportProvider.handleDocumentSymbol(params.document.uri, documentManager, RequestType.DocumentSymbol) }) @@ -361,6 +370,12 @@ export async function startServer (): Promise { connection.onDocumentHighlight(async params => { return await highlightSymbolProvider.handleDocumentHighlightRequest(params, documentManager) }) + + /** -------------- SEMANTIC TOKENS SUPPORT --------------- **/ + connection.onRequest(SemanticTokensRequest.method, async (params: SemanticTokensParams) => { + return await semanticTokensProvider.handleSemanticTokensRequest(params, documentManager) + }) + setupSemanticTokensRefresh(connection, documentIndexer) } /** -------------------- Helper Functions -------------------- **/ diff --git a/tests/providers/semanticTokens/SemanticTokensProvider.test.ts b/tests/providers/semanticTokens/SemanticTokensProvider.test.ts new file mode 100644 index 0000000..aa08fe8 --- /dev/null +++ b/tests/providers/semanticTokens/SemanticTokensProvider.test.ts @@ -0,0 +1,296 @@ +import assert from 'assert' +import sinon from 'sinon' +import quibble from 'quibble' + +import { TextDocuments, Range } from 'vscode-languageserver' +import { TextDocument } from 'vscode-languageserver-textdocument' + +import MatlabLifecycleManager from '../../../src/lifecycle/MatlabLifecycleManager' +import DocumentIndexer from '../../../src/indexing/DocumentIndexer' +import Indexer from '../../../src/indexing/Indexer' +import FileInfoIndex, { MatlabFunctionScopeInfo, MatlabGlobalScopeInfo } from '../../../src/indexing/FileInfoIndex' +import ClientConnection from '../../../src/ClientConnection' +import getMockConnection from '../../mocks/Connection.mock' +import getMockMvm from '../../mocks/Mvm.mock' +import { dynamicImport } from '../../TestUtils' + +describe('SemanticTokenProvider', () => { + const URI = 'file://test.m' + + let matlabLifecycleManager: MatlabLifecycleManager + let fileInfoIndex: FileInfoIndex + let indexer: Indexer + let documentIndexer: DocumentIndexer + + let semanticTokensProvider: any + let documentManager: TextDocuments + + const setup = () => { + const mockMvm = getMockMvm() + + matlabLifecycleManager = new MatlabLifecycleManager() + fileInfoIndex = new FileInfoIndex() + indexer = new Indexer(matlabLifecycleManager, mockMvm, fileInfoIndex) + documentIndexer = new DocumentIndexer(indexer, fileInfoIndex) + + documentManager = new TextDocuments(TextDocument) + + sinon.stub(matlabLifecycleManager, 'getMatlabConnection').resolves({} as any) + + type SemanticTokensProviderExports = typeof import('../../../src/providers/semanticTokens/SemanticTokensProvider') + const { default: SemanticTokensProvider } = dynamicImport( + module, '../../../src/providers/semanticTokens/SemanticTokensProvider') + + semanticTokensProvider = new SemanticTokensProvider( + matlabLifecycleManager, + documentIndexer, + fileInfoIndex + ) + + const doc = TextDocument.create( + URI, 'matlab', 1, 'abc' + ) + + sinon.stub(documentManager, 'get').returns(doc) + } + + const teardown = () => { + quibble.reset() + sinon.restore() + } + + before(() => { + ClientConnection._setConnection(getMockConnection()) + }) + + after(() => { + ClientConnection._clearConnection() + }) + + describe('#handleSemanticTokensRequest', () => { + beforeEach(() => setup()) + afterEach(() => teardown()) + + it('should return null if there is no MATLAB connection', async () => { + (matlabLifecycleManager.getMatlabConnection as sinon.SinonStub).resolves(null) + + const res = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri: URI } }, documentManager + ) + + assert.strictEqual(res, null, 'Result should be null when there is no MATLAB connection') + }) + + it('should return null if there is no document at the given URI', async () => { + (documentManager.get as sinon.SinonStub).returns(undefined) + + const res = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri: URI } }, documentManager + ) + + assert.strictEqual(res, null, 'Result should be null when there is no document at the given URI') + }) + + it('should return null if codeinfo is null', async () => { + fileInfoIndex.codeInfoCache.set(URI, null as any) + + const res = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri: URI } }, documentManager + ) + + assert.strictEqual(res, null, 'Result should be null when codeinfo is null') + }) + + it('should mark foo as a semantic function', async () => { + const uri = URI + + const range = { + start: { line: 0, character: 0 }, + end: { line: 0, character: 3 } // "foo" + } + + fileInfoIndex.codeInfoCache.set( + URI, + createCodeInfo({ + functions: [ + { name: 'foo', range: range } + ] + }) + ) + + const result = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri } }, + documentManager + ) + + assert.ok(result, 'Expected semantic tokens result') + + const [deltaLine, deltaStart, length, typeIndex] = result!.data + + assert.strictEqual(typeIndex, 0, 'Expected function token type') + assert.strictEqual(length, 3, 'Expected token length for "foo"') + }) + + it('should mark x as a semantic variable', async () => { + const uri = URI + + const range = { + start: { line: 0, character: 0 }, + end: { line: 0, character: 1 } + } + + fileInfoIndex.codeInfoCache.set( + URI, + createCodeInfo({ + variables: [ + { name: 'x', range: range } + ] + }) + ) + + const result = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri } }, + documentManager + ) + + assert.ok(result) + + const [deltaLine, deltaStart, length, typeIndex] = result!.data + + assert.strictEqual(typeIndex, 1) // variable + assert.strictEqual(length, 1) + }) + + it('should encode delta correctly for semantic tokens on same line', async () => { + const uri = URI + + const range1 = { + start: { line: 0, character: 1 }, + end: { line: 0, character: 2 } // "x" + } as unknown as Range + + const range2 = { + start: { line: 0, character: 5 }, + end: { line: 0, character: 6 } // "y" + } as unknown as Range + + fileInfoIndex.codeInfoCache.set( + URI, + createCodeInfo({ + variables: [ + { name: 'x', range: range1 }, + { name: 'y', range: range2 } + ] + }) + ) + + const result = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri } } as any, + documentManager + ) + + assert.ok(result, 'Expected semantic tokens result') + + // Token 1 + const [dl1, ds1, len1, type1] = result!.data.slice(0, 4) + + // Token 2 + const [dl2, ds2, len2, type2] = result!.data.slice(5, 9) + + // First token (absolute) + assert.strictEqual(dl1, 0) + assert.strictEqual(ds1, 1) + + // Second token: + // same line → deltaLine = 0 + assert.strictEqual(dl2, 0, 'Expected same line') + + // relative start: 5 - 1 = 4 + assert.strictEqual(ds2, 4, 'Expected relative deltaStart') + }) + + it('should encode delta correctly for semantic tokens on different lines', async () => { + const uri = URI + + const range1 = { + start: { line: 0, character: 0 }, + end: { line: 0, character: 1 } // "x" + } + + const range2 = { + start: { line: 2, character: 7 }, + end: { line: 2, character: 8 } // "y" + } + + fileInfoIndex.codeInfoCache.set( + URI, + createCodeInfo({ + variables: [ // Note: Variables are intentionally added in reverse order to test sorting of tokens + { name: 'y', range: range2 }, + { name: 'x', range: range1 } + ] + }) + ) + + const result = await semanticTokensProvider.handleSemanticTokensRequest( + { textDocument: { uri } } as any, + documentManager + ) + + assert.ok(result, 'Expected semantic tokens result') + + // Token 1 + const [dl1, ds1, len1, type1] = result!.data.slice(0, 4) + + // Token 2 + const [dl2, ds2, len2, type2] = result!.data.slice(5, 9) + + // First token: absolute position + assert.strictEqual(dl1, 0, 'Expected first token to have dl1 = 0, sorting should ensure this is the case') + assert.strictEqual(ds1, 0, 'Expected first token to have ds1 = 0, sorting should ensure this is the case') + + // Second token: + // line 2 - line 0 = 2 + assert.strictEqual(dl2, 2, 'Expected deltaLine = 2') + + // since new line → start should NOT be relative + assert.strictEqual(ds2, 7, 'Expected absolute start on new line') + }) + }) +}) + +function createCodeInfo({ + variables = [], + functions = [] +}: { + variables?: Array<{ name: string, range: Range }> + functions?: Array<{ name: string, range: Range }> +}) { + return { + globalScopeInfo: { + variables: new Map( + variables.map(v => [ + v.name, + { + references: [ + { components: [{ range: v.range }] } + ] + } + ]) + ), + + functionOrUnboundReferences: new Map( + functions.map(f => [ + f.name, + { + references: [ + { components: [{ range: f.range }] } + ] + } + ]) + ), + + functionScopes: new Map() + } + } as any +} \ No newline at end of file