Add basic semantic highlighting

src/indexing/DocumentIndexer.ts

@@ -12,6 +12,7 @@ const INDEXING_DELAY = 500 // Delay (in ms) after keystroke before attempting to
  */
 export default class DocumentIndexer {
     private readonly pendingFilesToIndex = new Map<string, NodeJS.Timeout>()
+    private onIndexed?: (uri: string) => void
 
     constructor (
         private readonly indexer: Indexer,
@@ -42,6 +43,7 @@ export default class DocumentIndexer {
      */
     indexDocument (textDocument: TextDocument): void {
         void this.indexer.indexDocument(textDocument)
+        this.onIndexed?.(textDocument.uri)
     }
 
     /**
@@ -73,5 +75,11 @@ export default class DocumentIndexer {
         if (!this.fileInfoIndex.codeInfoCache.has(uri)) {
             await this.indexer.indexDocument(textDocument)
         }
+
+        this.onIndexed?.(uri)
+    }
+
+    setOnIndexed (callback: (uri: string) => void): void {
+        this.onIndexed = callback
     }
 }

src/providers/semanticTokens/SemanticTokensProvider.ts

@@ -0,0 +1,136 @@
+import { SemanticTokens, SemanticTokensParams, TextDocuments, Range, Connection } from 'vscode-languageserver'
+import MatlabLifecycleManager from '../../lifecycle/MatlabLifecycleManager'
+import { TextDocument } from 'vscode-languageserver-textdocument'
+import FileInfoIndex, { MatlabFunctionScopeInfo, MatlabGlobalScopeInfo } from '../../indexing/FileInfoIndex'
+import DocumentIndexer from '../../indexing/DocumentIndexer'
+
+interface SemanticToken {
+    range: Range
+    typeIndex: number
+}
+
+/**
+ * Handles requests for semantic tokens for a document.
+ */
+class SemanticTokensProvider {
+    constructor (
+        protected readonly matlabLifecycleManager: MatlabLifecycleManager,
+        protected readonly documentIndexer: DocumentIndexer,
+        protected readonly fileInfoIndex: FileInfoIndex
+    ) { }
+
+    async handleSemanticTokensRequest (
+        params: SemanticTokensParams,
+        documentManager: TextDocuments<TextDocument>
+    ): Promise<SemanticTokens | null> {
+        // This provider will be called constantly, should not connect to MATLAB just because it was called
+        const matlabConnection = await this.matlabLifecycleManager.getMatlabConnection(false)
+        // If MATLAB is not connected, fall back to default highlighting
+        if (matlabConnection == null) return null
+
+        const textDocument = documentManager.get(params.textDocument.uri)
+        if (textDocument == null) return null
+
+        const codeInfo = this.fileInfoIndex.codeInfoCache.get(params.textDocument.uri)
+        if (codeInfo == null) return null
+
+        const tokens: SemanticToken[] = []
+        this.collectSemanticTokens(codeInfo.globalScopeInfo, tokens)
+
+        // Sort tokens by their position in the document (line and character)
+        // This is necessary to encode them using relative positions
+        tokens.sort((a, b) => {
+            const lineDiff = a.range.start.line - b.range.start.line
+            if (lineDiff !== 0) return lineDiff
+
+            return a.range.start.character - b.range.start.character
+        })
+
+        const data: number[] = []
+        let prevLine = 0
+        let prevStart = 0
+
+        // Encode semantic tokens using relative line and character positions
+        for (const token of tokens) {
+            const line = token.range.start.line
+            const start = token.range.start.character
+            const length = token.range.end.character - token.range.start.character
+
+            const deltaLine = line - prevLine
+            const deltaStart = deltaLine === 0 ? start - prevStart : start
+
+            data.push(deltaLine, deltaStart, length, token.typeIndex, 0)
+            prevLine = line
+            prevStart = start
+        }
+
+        return { data }
+    }
+
+    /**
+     * Recursively collects semantic tokens for a given scope and its nested scopes.
+     * Tokens are appended to 'tokens' in-place.
+    * @param scope The scope from which semantic tokens should be collected
+    * @param tokens The array to which collected semantic tokens are appended
+    */
+    private collectSemanticTokens (
+        scope: MatlabGlobalScopeInfo | MatlabFunctionScopeInfo,
+        tokens: SemanticToken[]
+    ): void {
+        // Variables: highlight only the first component as variable
+        for (const item of scope.variables.values()) {
+            for (const ref of item.references) {
+                tokens.push({ range: ref.components[0].range, typeIndex: 1 }) // variable
+            }
+        }
+
+        // Functions/unbound: highlight only the first component as function
+        for (const item of scope.functionOrUnboundReferences.values()) {
+            for (const ref of item.references) {
+                tokens.push({ range: ref.components[0].range, typeIndex: 0 }) // function
+            }
+        }
+
+        // Class scope
+        const classScope = (scope as MatlabGlobalScopeInfo).classScope;
+        if (classScope != null) {
+            for (const nestedFunc of classScope.functionScopes.values()) {
+                if (nestedFunc.functionScopeInfo != null) {
+                    this.collectSemanticTokens(nestedFunc.functionScopeInfo, tokens);
+                }
+            }
+        }
+
+        // Function scopes
+        for (const nestedFunc of scope.functionScopes.values()) {
+            if (nestedFunc.functionScopeInfo != null) {
+                this.collectSemanticTokens(nestedFunc.functionScopeInfo, tokens)
+            }
+        }
+    }
+}
+
+export const SEMANTIC_TOKEN_TYPES = ['function', 'variable']
+export const SEMANTIC_TOKEN_MODIFIERS: string[] = []
+export default SemanticTokensProvider
+
+/**
+ * Wires semantic token invalidation to document indexing.
+ *
+ * When indexing completes, this schedules a debounced refresh request
+ * so the client re-requests semantic tokens and updates highlighting.
+ */
+export function setupSemanticTokensRefresh (
+    connection: Connection,
+    documentIndexer: DocumentIndexer
+): void {
+    let refreshTimer: NodeJS.Timeout | undefined
+
+    documentIndexer.setOnIndexed(() => {
+        if (refreshTimer != null) clearTimeout(refreshTimer)
+
+        refreshTimer = setTimeout(() => {
+            void connection.sendRequest('workspace/semanticTokens/refresh')
+        }, 150)
+    })
+}

src/server.ts

@@ -1,7 +1,7 @@
 // Copyright 2022 - 2025 The MathWorks, Inc.
 
 import { TextDocument } from 'vscode-languageserver-textdocument'
-import { ClientCapabilities, InitializeParams, InitializeResult, TextDocuments } from 'vscode-languageserver/node'
+import { ClientCapabilities, InitializeParams, InitializeResult, TextDocuments, SemanticTokensRequest, SemanticTokensParams } from 'vscode-languageserver/node'
 import DocumentIndexer from './indexing/DocumentIndexer'
 import WorkspaceIndexer from './indexing/WorkspaceIndexer'
 import ConfigurationManager, { ConnectionTiming } from './lifecycle/ConfigurationManager'
@@ -22,6 +22,7 @@ import PathResolver from './providers/navigation/PathResolver'
 import Indexer from './indexing/Indexer'
 import RenameSymbolProvider from './providers/rename/RenameSymbolProvider'
 import HighlightSymbolProvider from './providers/highlighting/HighlightSymbolProvider'
+import SemanticTokensProvider, { SEMANTIC_TOKEN_TYPES, SEMANTIC_TOKEN_MODIFIERS, setupSemanticTokensRefresh } from './providers/semanticTokens/SemanticTokensProvider'
 import { RequestType } from './indexing/SymbolSearchService'
 import { cacheAndClearProxyEnvironmentVariables } from './utils/ProxyUtils'
 import MatlabDebugAdaptorServer from './debug/MatlabDebugAdaptorServer'
@@ -73,6 +74,7 @@ export async function startServer (): Promise<void> {
     const navigationSupportProvider = new NavigationSupportProvider(matlabLifecycleManager, fileInfoIndex, indexer, documentIndexer, pathResolver)
     const renameSymbolProvider = new RenameSymbolProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex)
     const highlightSymbolProvider = new HighlightSymbolProvider(matlabLifecycleManager, documentIndexer, indexer, fileInfoIndex)
+    const semanticTokensProvider = new SemanticTokensProvider(matlabLifecycleManager, documentIndexer, fileInfoIndex)
 
     let pathSynchronizer: PathSynchronizer | null
 
@@ -142,7 +144,14 @@ export async function startServer (): Promise<void> {
                 renameProvider: {
                     prepareProvider: true
                 },
-                documentHighlightProvider: true
+                documentHighlightProvider: true,
+                semanticTokensProvider: {
+                    legend: {
+                        tokenTypes: SEMANTIC_TOKEN_TYPES,
+                        tokenModifiers: SEMANTIC_TOKEN_MODIFIERS
+                    },
+                    full: true
+                }
             }
         }
 
@@ -264,7 +273,7 @@ export async function startServer (): Promise<void> {
         reportFileOpened(params.document)
         void lintingSupportProvider.lintDocument(params.document)
         void documentIndexer.indexDocument(params.document)
-        
+
         void navigationSupportProvider.handleDocumentSymbol(params.document.uri, documentManager, RequestType.DocumentSymbol)
     })
 
@@ -361,6 +370,12 @@ export async function startServer (): Promise<void> {
     connection.onDocumentHighlight(async params => {
         return await highlightSymbolProvider.handleDocumentHighlightRequest(params, documentManager)
     })
+
+    /** -------------- SEMANTIC TOKENS SUPPORT --------------- **/
+    connection.onRequest(SemanticTokensRequest.method, async (params: SemanticTokensParams) => {
+        return await semanticTokensProvider.handleSemanticTokensRequest(params, documentManager)
+    })
+    setupSemanticTokensRefresh(connection, documentIndexer)
 }
 
 /** -------------------- Helper Functions -------------------- **/