From 97b905a2b054c372252a6646e5649eb487487b36 Mon Sep 17 00:00:00 2001 From: Ethan Johnson Date: Tue, 26 Nov 2019 12:12:15 -0600 Subject: [PATCH 1/6] support semanticolor syntax highlighting --- src/tree-sitter-language-mode.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index cb7d4cccb92..e4580100125 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -1292,7 +1292,11 @@ class LayerHighlightIterator { this.treeCursor.nodeIsNamed ); const scopeName = applyLeafRules(value, this.treeCursor); - if (scopeName) { + const node = this.treeCursor.currentNode; + if (!node.childCount) { + return this.languageLayer.languageMode.grammar.idForScope(scopeName || this.languageLayer.grammar.scopeName, + node.text); + } else if (scopeName) { return this.languageLayer.languageMode.grammar.idForScope(scopeName); } } From 88ad0b1eb0f22a0523bea243da53bd94ea79998d Mon Sep 17 00:00:00 2001 From: Ethan Johnson Date: Tue, 26 Nov 2019 13:09:49 -0600 Subject: [PATCH 2/6] pass node --- src/tree-sitter-grammar.js | 3 +++ src/tree-sitter-language-mode.js | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tree-sitter-grammar.js b/src/tree-sitter-grammar.js index e7e5e431d07..e76b8aed45e 100644 --- a/src/tree-sitter-grammar.js +++ b/src/tree-sitter-grammar.js @@ -71,6 +71,9 @@ module.exports = class TreeSitterGrammar { } idForScope(scopeName) { + if (!scopeName) { + return undefined; + } let id = this.idsByScope[scopeName]; if (!id) { id = this.nextScopeId += 2; diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index e4580100125..490ff9bdc14 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -1294,8 +1294,7 @@ class LayerHighlightIterator { const scopeName = applyLeafRules(value, this.treeCursor); const node = this.treeCursor.currentNode; if (!node.childCount) { - return this.languageLayer.languageMode.grammar.idForScope(scopeName || this.languageLayer.grammar.scopeName, - node.text); + return this.languageLayer.languageMode.grammar.idForScope(scopeName, node); } else if (scopeName) { return this.languageLayer.languageMode.grammar.idForScope(scopeName); } From 9d9c2ee2fd6299b350a17ef8222e71b3f25d7add Mon Sep 17 00:00:00 2001 From: Ethan Johnson Date: Tue, 26 Nov 2019 14:56:34 -0600 Subject: [PATCH 3/6] just text --- src/tree-sitter-language-mode.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index 490ff9bdc14..e2180028316 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -1294,7 +1294,7 @@ class LayerHighlightIterator { const scopeName = applyLeafRules(value, this.treeCursor); const node = this.treeCursor.currentNode; if (!node.childCount) { - return this.languageLayer.languageMode.grammar.idForScope(scopeName, node); + return this.languageLayer.languageMode.grammar.idForScope(scopeName, node.text); } else if (scopeName) { return this.languageLayer.languageMode.grammar.idForScope(scopeName); } From 3191997c874dcf142eba58fbc83e3d77909fa127 Mon Sep 17 00:00:00 2001 From: Ethan Johnson Date: Mon, 15 Jun 2020 12:12:02 -0500 Subject: [PATCH 4/6] add test --- package-lock.json | 24 ++++++++---- script/package-lock.json | 3 +- spec/tree-sitter-language-mode-spec.js | 53 ++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index b3ee3115f0a..5b8f9a38f9d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1273,11 +1273,13 @@ "dependencies": { "etch": { "version": "0.9.0", - "resolved": false + "resolved": "https://registry.npmjs.org/etch/-/etch-0.9.0.tgz", + "integrity": "sha1-CSJpiPLO4GkL3yCMyyXkFNXfrV8=" }, "semver": { "version": "5.7.1", - "resolved": false + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" } } }, @@ -2419,7 +2421,8 @@ "dependencies": { "grim": { "version": "2.0.2", - "resolved": false, + "resolved": "https://registry.npmjs.org/grim/-/grim-2.0.2.tgz", + "integrity": "sha512-Qj7hTJRfd87E/gUgfvM0YIH/g2UA2SV6niv6BYXk1o6w4mhgv+QyYM1EjOJQljvzgEj4SqSsRWldXIeKHz3e3Q==", "requires": { "event-kit": "^2.0.0" } @@ -2585,11 +2588,13 @@ "dependencies": { "etch": { "version": "0.9.0", - "resolved": false + "resolved": "https://registry.npmjs.org/etch/-/etch-0.9.0.tgz", + "integrity": "sha1-CSJpiPLO4GkL3yCMyyXkFNXfrV8=" }, "grim": { "version": "2.0.2", - "resolved": false, + "resolved": "https://registry.npmjs.org/grim/-/grim-2.0.2.tgz", + "integrity": "sha512-Qj7hTJRfd87E/gUgfvM0YIH/g2UA2SV6niv6BYXk1o6w4mhgv+QyYM1EjOJQljvzgEj4SqSsRWldXIeKHz3e3Q==", "requires": { "event-kit": "^2.0.0" } @@ -3533,14 +3538,16 @@ "dependencies": { "rimraf": { "version": "2.6.3", - "resolved": false, + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", "requires": { "glob": "^7.1.3" } }, "temp": { "version": "0.8.4", - "resolved": false, + "resolved": "https://registry.npmjs.org/temp/-/temp-0.8.4.tgz", + "integrity": "sha512-s0ZZzd0BzYv5tLSptZooSjK8oj6C+c19p7Vqta9+6NPOf7r+fxq0cJe6/oN4LTC79sy5NY8ucOJNgwsKCSbfqg==", "requires": { "rimraf": "~2.6.2" } @@ -7478,7 +7485,8 @@ "dependencies": { "etch": { "version": "0.9.0", - "resolved": false + "resolved": "https://registry.npmjs.org/etch/-/etch-0.9.0.tgz", + "integrity": "sha1-CSJpiPLO4GkL3yCMyyXkFNXfrV8=" } } }, diff --git a/script/package-lock.json b/script/package-lock.json index 77bfc024887..55f21683dae 100644 --- a/script/package-lock.json +++ b/script/package-lock.json @@ -1122,8 +1122,7 @@ "boolean": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.0.1.tgz", - "integrity": "sha512-HRZPIjPcbwAVQvOTxR4YE3o8Xs98NqbbL1iEZDCz7CL8ql0Lt5iOyJFxfnAB0oFs8Oh02F/lLlg30Mexv46LjA==", - "optional": true + "integrity": "sha512-HRZPIjPcbwAVQvOTxR4YE3o8Xs98NqbbL1iEZDCz7CL8ql0Lt5iOyJFxfnAB0oFs8Oh02F/lLlg30Mexv46LjA==" }, "brace-expansion": { "version": "1.1.11", diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 501478e3256..787b7754217 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -73,6 +73,59 @@ describe('TreeSitterLanguageMode', () => { ]); }); + it('provides the grammar with the text of leaf nodes only', async () => { + const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { + parser: 'tree-sitter-javascript', + scopes: { + program: 'source', + 'call_expression > identifier': 'function', + property_identifier: 'property', + 'call_expression > member_expression > property_identifier': 'method' + } + }); + const original = grammar.idForScope.bind(grammar); + let tokens = []; + grammar.idForScope = function (scope, text) { + if (text && tokens[tokens.length - 1] !== text) { + tokens.push(text); + } + return original(scope, text); + } + + buffer.setText('aa.bbb = cc(d.eee());'); + + const languageMode = new TreeSitterLanguageMode({ buffer, grammar }); + buffer.setLanguageMode(languageMode); + + expectTokensToEqual(editor, [ + [ + { text: 'aa.', scopes: ['source'] }, + { text: 'bbb', scopes: ['source', 'property'] }, + { text: ' = ', scopes: ['source'] }, + { text: 'cc', scopes: ['source', 'function'] }, + { text: '(d.', scopes: ['source'] }, + { text: 'eee', scopes: ['source', 'method'] }, + { text: '());', scopes: ['source'] } + ] + ]); + + expect(tokens).toEqual([ + 'aa', + '.', + 'bbb', + ' ', + '=', + 'cc', + '(', + 'd', + '.', + 'eee', + '(', + ')', + ';' + ]); + }); + it('can start or end multiple scopes at the same position', async () => { const grammar = new TreeSitterGrammar(atom.grammars, jsGrammarPath, { parser: 'tree-sitter-javascript', From eca8bb615f3e73e584fc58cbe3574b0d6897340e Mon Sep 17 00:00:00 2001 From: Ethan Johnson Date: Mon, 15 Jun 2020 12:14:26 -0500 Subject: [PATCH 5/6] no whitespace --- spec/tree-sitter-language-mode-spec.js | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 787b7754217..5ce515d0e18 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -113,7 +113,6 @@ describe('TreeSitterLanguageMode', () => { 'aa', '.', 'bbb', - ' ', '=', 'cc', '(', From 33cefdf8cf418a6afbe19589846bbba413bd04a8 Mon Sep 17 00:00:00 2001 From: Ethan Johnson Date: Tue, 16 Jun 2020 09:59:29 -0500 Subject: [PATCH 6/6] lint --- spec/tree-sitter-language-mode-spec.js | 4 ++-- src/tree-sitter-language-mode.js | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/spec/tree-sitter-language-mode-spec.js b/spec/tree-sitter-language-mode-spec.js index 5ce515d0e18..18375a84ebb 100644 --- a/spec/tree-sitter-language-mode-spec.js +++ b/spec/tree-sitter-language-mode-spec.js @@ -85,12 +85,12 @@ describe('TreeSitterLanguageMode', () => { }); const original = grammar.idForScope.bind(grammar); let tokens = []; - grammar.idForScope = function (scope, text) { + grammar.idForScope = function(scope, text) { if (text && tokens[tokens.length - 1] !== text) { tokens.push(text); } return original(scope, text); - } + }; buffer.setText('aa.bbb = cc(d.eee());'); diff --git a/src/tree-sitter-language-mode.js b/src/tree-sitter-language-mode.js index e2180028316..cf296c250ab 100644 --- a/src/tree-sitter-language-mode.js +++ b/src/tree-sitter-language-mode.js @@ -1294,7 +1294,10 @@ class LayerHighlightIterator { const scopeName = applyLeafRules(value, this.treeCursor); const node = this.treeCursor.currentNode; if (!node.childCount) { - return this.languageLayer.languageMode.grammar.idForScope(scopeName, node.text); + return this.languageLayer.languageMode.grammar.idForScope( + scopeName, + node.text + ); } else if (scopeName) { return this.languageLayer.languageMode.grammar.idForScope(scopeName); }