[pull] upstream_master from atom:master

spec/tree-sitter-language-mode-spec.js

@@ -73,6 +73,58 @@ describe('TreeSitterLanguageMode', () => {
       ]);
     });
 
+    it('provides the grammar with the text of leaf nodes only', async () => {
+      const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
+        parser: 'tree-sitter-javascript',
+        scopes: {
+          program: 'source',
+          'call_expression > identifier': 'function',
+          property_identifier: 'property',
+          'call_expression > member_expression > property_identifier': 'method'
+        }
+      });
+      const original = grammar.idForScope.bind(grammar);
+      let tokens = [];
+      grammar.idForScope = function(scope, text) {
+        if (text && tokens[tokens.length - 1] !== text) {
+          tokens.push(text);
+        }
+        return original(scope, text);
+      };
+
+      buffer.setText('aa.bbb = cc(d.eee());');
+
+      const languageMode = new TreeSitterLanguageMode({ buffer, grammar });
+      buffer.setLanguageMode(languageMode);
+
+      expectTokensToEqual(editor, [
+        [
+          { text: 'aa.', scopes: ['source'] },
+          { text: 'bbb', scopes: ['source', 'property'] },
+          { text: ' = ', scopes: ['source'] },
+          { text: 'cc', scopes: ['source', 'function'] },
+          { text: '(d.', scopes: ['source'] },
+          { text: 'eee', scopes: ['source', 'method'] },
+          { text: '());', scopes: ['source'] }
+        ]
+      ]);
+
+      expect(tokens).toEqual([
+        'aa',
+        '.',
+        'bbb',
+        '=',
+        'cc',
+        '(',
+        'd',
+        '.',
+        'eee',
+        '(',
+        ')',
+        ';'
+      ]);
+    });
+
     it('can start or end multiple scopes at the same position', async () => {
       const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, {
         parser: 'tree-sitter-javascript',

src/tree-sitter-grammar.js

@@ -71,6 +71,9 @@ module.exports = class TreeSitterGrammar {
   }
 
   idForScope(scopeName) {
+    if (!scopeName) {
+      return undefined;
+    }
     let id = this.idsByScope[scopeName];
     if (!id) {
       id = this.nextScopeId += 2;

src/tree-sitter-language-mode.js

@@ -1292,7 +1292,13 @@ class LayerHighlightIterator {
       this.treeCursor.nodeIsNamed
     );
     const scopeName = applyLeafRules(value, this.treeCursor);
-    if (scopeName) {
+    const node = this.treeCursor.currentNode;
+    if (!node.childCount) {
+      return this.languageLayer.languageMode.grammar.idForScope(
+        scopeName,
+        node.text
+      );
+    } else if (scopeName) {
       return this.languageLayer.languageMode.grammar.idForScope(scopeName);
     }
   }