diff --git a/locate/locate.js b/locate/locate.js index 96498f4f0..ce75c2c1a 100644 --- a/locate/locate.js +++ b/locate/locate.js @@ -4,6 +4,11 @@ const locator = require('ruby-method-locate'), const fs = require('fs'), path = require('path'); const _ = require('lodash'); +const async = require('async'); + +// vague values +const WALK_CONCURRENCY = 8; +const SINGLE_FILE_PARSE_CONCURRENCY = 8; const DECLARATION_TYPES = ['class', 'module', 'method', 'classMethod']; @@ -31,13 +36,13 @@ function flatten(locateInfo, file, containerName = '') { }); } function camelCaseRegExp(query) { - const escaped = _.escapeRegExp(query) + const escaped = _.escapeRegExp(query); const prefix = escaped.charAt(0); return new RegExp( `[${prefix.toLowerCase()}${prefix.toUpperCase()}]` + escaped.slice(1).replace(/[A-Z]|([a-z])/g, (char, lower) => { if (lower) return `[${char.toUpperCase()}${char}]`; - const lowered = char.toLowerCase() + const lowered = char.toLowerCase(); return `.*(?:${char}|_${lowered})`; }) ); @@ -45,7 +50,7 @@ function camelCaseRegExp(query) { function filter(symbols, query, matcher) { // TODO: Ask MS to expose or separate matchesFuzzy method. // https://github.com/Microsoft/vscode/blob/a1d3c8a3006d0a3d68495122ea09a2a37bca69db/src/vs/base/common/filters.ts - const isLowerCase = (query.toLowerCase() === query) + const isLowerCase = (query.toLowerCase() === query); const exact = new RegExp('^' + _.escapeRegExp(query) + '$', 'i'); const prefix = new RegExp('^' + _.escapeRegExp(query), 'i'); const substring = new RegExp(_.escapeRegExp(query), isLowerCase ? 'i' : ''); @@ -60,17 +65,18 @@ module.exports = class Locate { this.settings = settings; this.root = root; this.tree = {}; - // begin the build ... - this.walk(this.root); - // add edit hooks - // always: do this file now (if it's in the tree) - // add lookup hooks + this.walkQueue = this._createWalkQueue(); + this.walkPromise = null; + this.parseQueue = async.queue((task, callback) => task().then(() => callback), SINGLE_FILE_PARSE_CONCURRENCY); } listInFile(absPath) { const waitForParse = (absPath in this.tree) ? Promise.resolve() : this.parse(absPath); return waitForParse.then(() => _.clone(this.tree[absPath] || [])); } find(name) { + return this._waitForWalk().then(() => this._find(name)); + } + _find(name) { // because our word pattern is designed to match symbols // things like Gem::RequestSet may request a search for ':RequestSet' const escapedName = _.escapeRegExp(_.trimStart(name, ':')); @@ -83,6 +89,9 @@ module.exports = class Locate { .value(); } query(query) { + return this._waitForWalk().then(() => this._query(query)); + } + _query(query) { const segmentMatch = query.match(/^(?:([^.#:]+)(.|#|::))([^.#:]+)$/) || []; const containerQuery = segmentMatch[1]; const separator = segmentMatch[2]; @@ -103,47 +112,76 @@ module.exports = class Locate { const nameMatches = filter(symbols, nameQuery, (symbolInfo, regexp) => { return _.includes(segmentTypes, symbolInfo.type) && regexp.test(symbolInfo.name); }); - const containerMatches = filter(nameMatches, containerQuery, (symbolInfo, regexp) => regexp.test(symbolInfo.containerName)) + const containerMatches = filter(nameMatches, containerQuery, (symbolInfo, regexp) => regexp.test(symbolInfo.containerName)); return _.uniq(plainMatches.concat(containerMatches)); } rm(absPath) { if (absPath in this.tree) delete this.tree[absPath]; } parse(absPath) { - const relPath = path.relative(this.root, absPath); - if (this.settings.exclude && minimatch(relPath, this.settings.exclude)) return; - if (this.settings.include && !minimatch(relPath, this.settings.include)) return; + this.parseQueue.push(() => this._parseAsync(absPath)); + } + _parseAsync(absPath, relPath) { + relPath = relPath || path.relative(this.root, absPath); + if (this.settings.exclude && minimatch(relPath, this.settings.exclude)) return Promise.resolve(); + if (this.settings.include && !minimatch(relPath, this.settings.include)) return Promise.resolve(); return locator(absPath) .then(result => { this.tree[absPath] = result ? flatten(result, absPath) : []; - }, err => { - if (err.code === 'EMFILE') { - // if there are too many open files - // try again after somewhere between 0 & 50 milliseconds - setTimeout(this.parse.bind(this, absPath), Math.random() * 50); - } else { - // otherwise, report it + }) + .catch(err => { + if (err.code !== 'ENOENT') { + // Maybe the file has already been removed while queued. console.log(err); - this.rm(absPath); } + this.rm(absPath); }); } - walk(root) { - fs.readdir(root, (err, files) => { - if (err) return; + walk() { + const startTime = new Date().getTime(); + this.walkPromise = new Promise((resolve, reject) => { + this.walkQueue.drain(); + this.walkQueue.kill(); + this.walkQueue = this._createWalkQueue(); + this.walkQueue.drain = () => resolve(); + this.walkQueue.push(callback => this._walkDir(this.root, err => err ? callback(err) : callback())); + }).then(() => console.log(`[ruby] locate: walk completed! (${new Date().getTime() - startTime}msec)`)); + return this.walkPromise; + } + _walkDir(dir, callback) { + fs.readdir(dir, (err, files) => { + if (err) { + callback(err); + return; + } files.forEach(file => { - const absPath = path.join(root, file); - const relPath = path.relative(this.root, absPath); - fs.stat(absPath, (err, stats) => { - if (err) return; - if (stats.isDirectory()) { - if (this.settings.exclude && minimatch(relPath, this.settings.exclude)) return; - this.walk(absPath); - } else { - this.parse(absPath); - } - }); + this.walkQueue.push(callback => this._walkNode(dir, file, callback)); }); + callback(); }); } + _walkNode(dir, file, callback) { + const absPath = path.join(dir, file); + const relPath = path.relative(this.root, absPath); + fs.stat(absPath, (err, stats) => { + if (err) { + callback(err); + return; + } + if (stats.isDirectory()) { + if (!(this.settings.exclude && minimatch(relPath, this.settings.exclude))) { + this.walkQueue.push(callback => this._walkDir(absPath, callback)); + } + callback(); + } else { + this._parseAsync(absPath, relPath).then(() => callback()); + } + }); + } + _createWalkQueue() { + return async.queue((task, callback) => task(callback), WALK_CONCURRENCY); + } + _waitForWalk() { + return this.walkPromise || this.walk(); + } }; diff --git a/package.json b/package.json index 0ff126b73..39fcd367b 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,8 @@ "vscode-debugadapter": "~1.6.0-pre8", "xmldom": "^0.1.19", "lodash": "^4.17.3", - "minimatch": "^3.0.3" + "minimatch": "^3.0.3", + "async": "^2.3.0" }, "devDependencies": { "gulp": "^3.9.0", @@ -77,6 +78,10 @@ "language": "erb", "path": "./snippets/erb.json" }], + "commands": [{ + "command": "ruby.reloadProject", + "title": "Ruby: Reload Project" + }], "configuration": { "title": "ruby language settings", "properties": { diff --git a/ruby.js b/ruby.js index 314219991..009a563ce 100644 --- a/ruby.js +++ b/ruby.js @@ -162,19 +162,25 @@ function activate(context) { vscode.window.visibleTextEditors.forEach(changeTrigger); //for locate: if it's a project, use the root, othewise, don't bother - let locate; if (vscode.workspace.rootPath) { + const refreshLocate = () => { + let progressOptions = { location: vscode.ProgressLocation.Window, title: 'Indexing Ruby source files' }; + vscode.window.withProgress(progressOptions, () => locate.walk()); + }; const settings = vscode.workspace.getConfiguration("ruby.locate") || {}; - locate = new Locate(vscode.workspace.rootPath, settings); + let locate = new Locate(vscode.workspace.rootPath, settings); + refreshLocate(); + subs.push(vscode.commands.registerCommand('ruby.reloadProject', refreshLocate)); + const watch = vscode.workspace.createFileSystemWatcher(settings.include); watch.onDidChange(uri => locate.parse(uri.fsPath)); watch.onDidCreate(uri => locate.parse(uri.fsPath)); watch.onDidDelete(uri => locate.rm(uri.fsPath)); + const locationConverter = match => new vscode.Location(vscode.Uri.file(match.file), new vscode.Position(match.line, match.char)); const defProvider = { provideDefinition: (doc, pos) => { const txt = doc.getText(doc.getWordRangeAtPosition(pos)); - const matches = locate.find(txt); - return matches.map(m => new vscode.Location(vscode.Uri.file(m.file), new vscode.Position(m.line, m.char))); + return locate.find(txt).then(matches => matches.map(locationConverter)); } }; subs.push(vscode.languages.registerDefinitionProvider(['ruby', 'erb'], defProvider)); @@ -191,21 +197,19 @@ function activate(context) { // NOTE: Workaround for high CPU usage on IPC (channel.onread) when too many symbols returned. // For channel.onread see issue like this: https://github.com/Microsoft/vscode/issues/6026 const numOfSymbolLimit = 3000; - const symbolConverter = matches => matches.slice(0, numOfSymbolLimit).map(match => { + const symbolsConverter = matches => matches.slice(0, numOfSymbolLimit).map(match => { const symbolKind = (symbolKindTable[match.type] || defaultSymbolKind)(match); - const uri = vscode.Uri.file(match.file); - const location = new Location(uri, new Position(match.line, match.char)); - return new SymbolInformation(match.name, symbolKind, match.containerName, location); + return new SymbolInformation(match.name, symbolKind, match.containerName, locationConverter(match)); }); const docSymbolProvider = { provideDocumentSymbols: (document, token) => { - return locate.listInFile(document.fileName).then(symbolConverter); + return locate.listInFile(document.fileName).then(symbolsConverter); } }; subs.push(vscode.languages.registerDocumentSymbolProvider(['ruby', 'erb'], docSymbolProvider)); const workspaceSymbolProvider = { provideWorkspaceSymbols: (query, token) => { - return symbolConverter(locate.query(query)); + return locate.query(query).then(symbolsConverter); } }; subs.push(vscode.languages.registerWorkspaceSymbolProvider(workspaceSymbolProvider));