Grammar tracking improvements

package.json

@@ -46,34 +46,37 @@
     "url": "https://github.com/rubyide/vscode-ruby/issues"
   },
   "dependencies": {
-    "vscode": "^1.1.4",
-    "ruby-method-locate": "*",
-    "vscode-debugprotocol": "^1.19.0",
-    "vscode-debugadapter": "^1.19.0",
-    "xmldom": "^0.1.19",
+    "async": "^2.3.0",
     "lodash": "^4.17.3",
     "minimatch": "^3.0.3",
-    "async": "^2.3.0"
+    "ruby-method-locate": "*",
+    "vscode": "^1.1.4",
+    "vscode-debugadapter": "^1.19.0",
+    "vscode-debugprotocol": "^1.19.0",
+    "xmldom": "^0.1.19"
   },
   "devDependencies": {
+    "@types/mocha": "^2.2.33",
+    "@types/node": "^6.0.50",
+    "cson-parser": "^2.0.1",
     "gulp": "^3.9.0",
-    "gulp-util": "^3.0.5",
-    "gulp-typescript": "^2.12.0",
     "gulp-sourcemaps": "^1.6.0",
-    "run-sequence": "*",
+    "gulp-typescript": "^2.12.0",
+    "gulp-util": "^3.0.5",
     "mocha": "^2.4.5",
-    "vscode-debugadapter-testsupport": "^1.19.0",
+    "run-sequence": "*",
     "typescript": "^2.1.5",
     "vscode": "^1.1.4",
-    "@types/mocha": "^2.2.33",
-    "@types/node": "^6.0.50"
+    "vscode-debugadapter-testsupport": "^1.19.0"
   },
   "scripts": {
     "vscode:prepublish": "tsc -p ./src",
     "compile": "tsc -p ./src",
     "watch": "tsc -w -p ./src",
     "test-debugger": "node ./node_modules/mocha/bin/mocha --timeout 15000 -u tdd ./out/debugger/tests/*.js",
-    "postinstall": "node ./node_modules/vscode/bin/install"
+    "postinstall": "node ./node_modules/vscode/bin/install",
+    "update-all-grammars": "node scripts/update-all-grammars",
+    "update-grammar": "node scripts/update-grammar atom/language-ruby"
   },
   "activationEvents": [
     "onLanguage:ruby",
@@ -324,12 +327,12 @@
       {
         "language": "ruby",
         "scopeName": "source.ruby",
-        "path": "./syntaxes/ruby.tmLanguage"
+        "path": "./syntaxes/ruby.cson.json"
       },
       {
         "language": "erb",
         "scopeName": "text.html.erb",
-        "path": "./syntaxes/erb.tmLanguage"
+        "path": "./syntaxes/erb.cson.json"
       }
     ],
     "debuggers": [

scripts/update-all-grammars.js

@@ -0,0 +1,28 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *
+ *  Modified for use in vscode-ruby
+ *  Original file here: https://github.com/Microsoft/vscode/blob/master/build/npm/update-all-grammars.js
+ *--------------------------------------------------------------------------------------------*/
+
+const cp = require('child_process');
+const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm';
+
+function updateGrammar(grammar) {
+	const result = cp.spawnSync(npm, ['run', 'update-grammar', grammar], {
+		stdio: 'inherit'
+	});
+
+	if (result.error || result.status !== 0) {
+		process.exit(1);
+	}
+}
+
+const extensions = [
+	'erb',
+	'gemfile',
+	'ruby'
+];
+
+extensions.forEach(extension => updateGrammar(extension));

scripts/update-grammar.js

@@ -0,0 +1,149 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *
+ *  Modified for use in vscode-ruby
+ *  Original file here: https://github.com/Microsoft/vscode/blob/master/build/npm/update-grammar.js
+ *--------------------------------------------------------------------------------------------*/
+
+'use strict';
+
+var path = require('path');
+var fs = require('fs');
+var cson = require('cson-parser');
+var https = require('https');
+var url = require('url');
+
+function getOptions(urlString) {
+	var _url = url.parse(urlString);
+	var headers = {
+		'User-Agent': 'VSCode'
+	};
+	var token = process.env['GITHUB_TOKEN'];
+	if (token) {
+		headers['Authorization'] = 'token ' + token
+	}
+	return {
+		protocol: _url.protocol,
+		host: _url.host,
+		port: _url.port,
+		path: _url.path,
+		headers: headers
+	};
+}
+
+function download(url, redirectCount) {
+	return new Promise((c, e) => {
+		var content = '';
+		https.get(getOptions(url), function (response) {
+			response.on('data', function (data) {
+				content += data.toString();
+			}).on('end', function () {
+				let count = redirectCount || 0;
+				if (count < 5 && response.statusCode >= 300 && response.statusCode <= 303 || response.statusCode === 307) {
+					let location = response.headers['location'];
+					if (location) {
+						console.log("Redirected " + url + " to " + location);
+						download(location, count+1).then(c, e);
+						return;
+					}
+				}
+				c(content);
+			});
+		}).on('error', function (err) {
+			e(err.message);
+		});
+	});
+}
+
+function grammarToTmLanguage(grammar) {
+	grammar = grammar.toLowerCase();
+	switch(grammar) {
+		case 'ruby':
+			var file = 'ruby.cson';
+			break;
+		case 'gemfile':
+			file = 'gemfile.cson';
+			break;
+		case 'erb':
+			file = 'html (ruby - erb).cson';
+			break;
+	}
+
+	return 'grammars/' + file;
+}
+
+function getCommitSha(repoId, repoPath) {
+	var commitInfo = 'https://api.github.com/repos/' + repoId + '/commits?path=' + repoPath;
+	return download(commitInfo).then(function (content) {
+		try {
+			let lastCommit = JSON.parse(content)[0];
+			return Promise.resolve({
+				commitSha: lastCommit.sha,
+				commitDate: lastCommit.commit.author.date
+			});
+		} catch (e) {
+			console.error("Failed extracting the SHA: " + content);
+			return Promise.resolve(null);
+		}
+	}, function () {
+		console.error('Failed loading ' + commitInfo);
+		return Promise.resolve(null);
+	});
+}
+
+exports.update = function (repoId, repoPath, dest, modifyGrammar, version = 'master') {
+	var contentPath = 'https://raw.githubusercontent.com/' + repoId + `/${version}/` + repoPath;
+	console.log('Reading from ' + contentPath);
+	return download(contentPath).then(function (content) {
+		var ext = path.extname(repoPath);
+		var grammar;
+		if (ext === '.cson') {
+			grammar = cson.parse(content);
+		} else {
+			console.error('Unknown file extension: ' + ext);
+			return;
+		}
+		if (modifyGrammar) {
+			modifyGrammar(grammar);
+		}
+		return getCommitSha(repoId, repoPath).then(function (info) {
+			let result = {
+				information_for_contributors: [
+					'This file has been converted from https://github.com/' + repoId + '/blob/master/' + repoPath,
+					'If you want to provide a fix or improvement, please create a pull request against the original repository.',
+					'Once accepted there, we are happy to receive an update request.'
+				]
+			};
+
+			if (info) {
+				result.version = 'https://github.com/' + repoId + '/commit/' + info.commitSha;
+			}
+			for (let key in grammar) {
+				if (!result.hasOwnProperty(key)) {
+					result[key] = grammar[key];
+				}
+			}
+
+			try {
+				fs.writeFileSync(dest, JSON.stringify(result, null, '\t'));
+				if (info) {
+					console.log('Updated ' + path.basename(dest) + ' to ' + repoId + '@' + info.commitSha.substr(0, 7) + ' (' + info.commitDate.substr(0, 10) + ')');
+				} else {
+					console.log('Updated ' + path.basename(dest));
+				}
+			} catch (e) {
+				console.error(e);
+			}
+		});
+
+	}, console.error);
+};
+
+if (path.basename(process.argv[1]).indexOf('update-grammar') !== -1) {
+	let repo = process.argv[2];
+	let grammar = process.argv[3];
+	let outputFile = path.join('syntaxes', grammar + '.cson.json');
+	let repoFile = grammarToTmLanguage(grammar);
+	exports.update(repo, repoFile, outputFile);
+}