Completely fix too many open files issue by using queue

locate/locate.js

@@ -4,6 +4,11 @@ const locator = require('ruby-method-locate'),
 const fs = require('fs'),
 	path = require('path');
 const _ = require('lodash');
+const async = require('async');
+
+// vague values
+const WALK_CONCURRENCY = 8;
+const SINGLE_FILE_PARSE_CONCURRENCY = 8;
 
 const DECLARATION_TYPES = ['class', 'module', 'method', 'classMethod'];
 
@@ -31,21 +36,21 @@ function flatten(locateInfo, file, containerName = '') {
 	});
 }
 function camelCaseRegExp(query) {
-	const escaped = _.escapeRegExp(query)
+	const escaped = _.escapeRegExp(query);
 	const prefix = escaped.charAt(0);
 	return new RegExp(
 		`[${prefix.toLowerCase()}${prefix.toUpperCase()}]` +
 		escaped.slice(1).replace(/[A-Z]|([a-z])/g, (char, lower) => {
 			if (lower) return `[${char.toUpperCase()}${char}]`;
-			const lowered = char.toLowerCase()
+			const lowered = char.toLowerCase();
 			return `.*(?:${char}|_${lowered})`;
 		})
 	);
 }
 function filter(symbols, query, matcher) {
 	// TODO: Ask MS to expose or separate matchesFuzzy method.
 	// https://github.com/Microsoft/vscode/blob/a1d3c8a3006d0a3d68495122ea09a2a37bca69db/src/vs/base/common/filters.ts
-	const isLowerCase = (query.toLowerCase() === query)
+	const isLowerCase = (query.toLowerCase() === query);
 	const exact = new RegExp('^' + _.escapeRegExp(query) + '$', 'i');
 	const prefix = new RegExp('^' + _.escapeRegExp(query), 'i');
 	const substring = new RegExp(_.escapeRegExp(query), isLowerCase ? 'i' : '');
@@ -60,17 +65,18 @@ module.exports = class Locate {
 		this.settings = settings;
 		this.root = root;
 		this.tree = {};
-		// begin the build ...
-		this.walk(this.root);
-		// add edit hooks
-		// always: do this file now (if it's in the tree)
-		// add lookup hooks
+		this.walkQueue = this._createWalkQueue();
+		this.walkPromise = null;
+		this.parseQueue = async.queue((task, callback) => task().then(() => callback), SINGLE_FILE_PARSE_CONCURRENCY);
 	}
 	listInFile(absPath) {
 		const waitForParse = (absPath in this.tree) ? Promise.resolve() : this.parse(absPath);
 		return waitForParse.then(() => _.clone(this.tree[absPath] || []));
 	}
 	find(name) {
+		return this._waitForWalk().then(() => this._find(name));
+	}
+	_find(name) {
 		// because our word pattern is designed to match symbols
 		// things like Gem::RequestSet may request a search for ':RequestSet'
 		const escapedName = _.escapeRegExp(_.trimStart(name, ':'));
@@ -83,6 +89,9 @@ module.exports = class Locate {
 			.value();
 	}
 	query(query) {
+		return this._waitForWalk().then(() => this._query(query));
+	}
+	_query(query) {
 		const segmentMatch = query.match(/^(?:([^.#:]+)(.|#|::))([^.#:]+)$/) || [];
 		const containerQuery = segmentMatch[1];
 		const separator = segmentMatch[2];
@@ -103,47 +112,76 @@ module.exports = class Locate {
 		const nameMatches = filter(symbols, nameQuery, (symbolInfo, regexp) => {
 			return _.includes(segmentTypes, symbolInfo.type) && regexp.test(symbolInfo.name);
 		});
-		const containerMatches = filter(nameMatches, containerQuery, (symbolInfo, regexp) => regexp.test(symbolInfo.containerName))
+		const containerMatches = filter(nameMatches, containerQuery, (symbolInfo, regexp) => regexp.test(symbolInfo.containerName));
 		return _.uniq(plainMatches.concat(containerMatches));
 	}
 	rm(absPath) {
 		if (absPath in this.tree) delete this.tree[absPath];
 	}
 	parse(absPath) {
-		const relPath = path.relative(this.root, absPath);
-		if (this.settings.exclude && minimatch(relPath, this.settings.exclude)) return;
-		if (this.settings.include && !minimatch(relPath, this.settings.include)) return;
+		this.parseQueue.push(() => this._parseAsync(absPath));
+	}
+	_parseAsync(absPath, relPath) {
+		relPath = relPath || path.relative(this.root, absPath);
+		if (this.settings.exclude && minimatch(relPath, this.settings.exclude)) return Promise.resolve();
+		if (this.settings.include && !minimatch(relPath, this.settings.include)) return Promise.resolve();
 		return locator(absPath)
 			.then(result => {
 				this.tree[absPath] = result ? flatten(result, absPath) : [];
-			}, err => {
-				if (err.code === 'EMFILE') {
-					// if there are too many open files
-					// try again after somewhere between 0 & 50 milliseconds
-					setTimeout(this.parse.bind(this, absPath), Math.random() * 50);
-				} else {
-					// otherwise, report it
+			})
+			.catch(err => {
+				if (err.code !== 'ENOENT') {
+					// Maybe the file has already been removed while queued.
 					console.log(err);
-					this.rm(absPath);
 				}
+				this.rm(absPath);
 			});
 	}
-	walk(root) {
-		fs.readdir(root, (err, files) => {
-			if (err) return;
+	walk() {
+		const startTime = new Date().getTime();
+		this.walkPromise = new Promise((resolve, reject) => {
+			this.walkQueue.drain();
+			this.walkQueue.kill();
+			this.walkQueue = this._createWalkQueue();
+			this.walkQueue.drain = () => resolve();
+			this.walkQueue.push(callback => this._walkDir(this.root, err => err ? callback(err) : callback()));
+		}).then(() => console.log(`[ruby] locate: walk completed! (${new Date().getTime() - startTime}msec)`));
+		return this.walkPromise;
+	}
+	_walkDir(dir, callback) {
+		fs.readdir(dir, (err, files) => {
+			if (err) {
+				callback(err);
+				return;
+			}
 			files.forEach(file => {
-				const absPath = path.join(root, file);
-				const relPath = path.relative(this.root, absPath);
-				fs.stat(absPath, (err, stats) => {
-					if (err) return;
-					if (stats.isDirectory()) {
-						if (this.settings.exclude && minimatch(relPath, this.settings.exclude)) return;
-						this.walk(absPath);
-					} else {
-						this.parse(absPath);
-					}
-				});
+				this.walkQueue.push(callback => this._walkNode(dir, file, callback));
 			});
+			callback();
 		});
 	}
+	_walkNode(dir, file, callback) {
+		const absPath = path.join(dir, file);
+		const relPath = path.relative(this.root, absPath);
+		fs.stat(absPath, (err, stats) => {
+			if (err) {
+				callback(err);
+				return;
+			}
+			if (stats.isDirectory()) {
+				if (!(this.settings.exclude && minimatch(relPath, this.settings.exclude))) {
+					this.walkQueue.push(callback => this._walkDir(absPath, callback));
+				}
+				callback();
+			} else {
+				this._parseAsync(absPath, relPath).then(() => callback());
+			}
+		});
+	}
+	_createWalkQueue() {
+		return async.queue((task, callback) => task(callback), WALK_CONCURRENCY);
+	}
+	_waitForWalk() {
+		return this.walkPromise || this.walk();
+	}
 };

package.json

@@ -46,7 +46,8 @@
 		"vscode-debugadapter": "~1.6.0-pre8",
 		"xmldom": "^0.1.19",
 		"lodash": "^4.17.3",
-		"minimatch": "^3.0.3"
+		"minimatch": "^3.0.3",
+		"async": "^2.3.0"
 	},
 	"devDependencies": {
 		"gulp": "^3.9.0",
@@ -77,6 +78,10 @@
 			"language": "erb",
 			"path": "./snippets/erb.json"
 		}],
+		"commands": [{
+			"command": "ruby.reloadProject",
+			"title": "Ruby: Reload Project"
+		}],
 		"configuration": {
 			"title": "ruby language settings",
 			"properties": {

ruby.js

@@ -162,19 +162,25 @@ function activate(context) {
 	vscode.window.visibleTextEditors.forEach(changeTrigger);
 
 	//for locate: if it's a project, use the root, othewise, don't bother
-	let locate;
 	if (vscode.workspace.rootPath) {
+		const refreshLocate = () => {
+			let progressOptions = { location: vscode.ProgressLocation.Window, title: 'Indexing Ruby source files' };
+			vscode.window.withProgress(progressOptions, () => locate.walk());
+		};
 		const settings = vscode.workspace.getConfiguration("ruby.locate") || {};
-		locate = new Locate(vscode.workspace.rootPath, settings);
+		let locate = new Locate(vscode.workspace.rootPath, settings);
+		refreshLocate();
+		subs.push(vscode.commands.registerCommand('ruby.reloadProject', refreshLocate));
+
 		const watch = vscode.workspace.createFileSystemWatcher(settings.include);
 		watch.onDidChange(uri => locate.parse(uri.fsPath));
 		watch.onDidCreate(uri => locate.parse(uri.fsPath));
 		watch.onDidDelete(uri => locate.rm(uri.fsPath));
+		const locationConverter = match => new vscode.Location(vscode.Uri.file(match.file), new vscode.Position(match.line, match.char));
 		const defProvider = {
 			provideDefinition: (doc, pos) => {
 				const txt = doc.getText(doc.getWordRangeAtPosition(pos));
-				const matches = locate.find(txt);
-				return matches.map(m => new vscode.Location(vscode.Uri.file(m.file), new vscode.Position(m.line, m.char)));
+				return locate.find(txt).then(matches => matches.map(locationConverter));
 			}
 		};
 		subs.push(vscode.languages.registerDefinitionProvider(['ruby', 'erb'], defProvider));
@@ -191,21 +197,19 @@ function activate(context) {
 		// NOTE: Workaround for high CPU usage on IPC (channel.onread) when too many symbols returned.
 		// For channel.onread see issue like this: https://github.com/Microsoft/vscode/issues/6026
 		const numOfSymbolLimit = 3000;
-		const symbolConverter = matches => matches.slice(0, numOfSymbolLimit).map(match => {
+		const symbolsConverter = matches => matches.slice(0, numOfSymbolLimit).map(match => {
 			const symbolKind = (symbolKindTable[match.type] || defaultSymbolKind)(match);
-			const uri = vscode.Uri.file(match.file);
-			const location = new Location(uri, new Position(match.line, match.char));
-			return new SymbolInformation(match.name, symbolKind, match.containerName, location);
+			return new SymbolInformation(match.name, symbolKind, match.containerName, locationConverter(match));
 		});
 		const docSymbolProvider = {
 			provideDocumentSymbols: (document, token) => {
-				return locate.listInFile(document.fileName).then(symbolConverter);
+				return locate.listInFile(document.fileName).then(symbolsConverter);
 			}
 		};
 		subs.push(vscode.languages.registerDocumentSymbolProvider(['ruby', 'erb'], docSymbolProvider));
 		const workspaceSymbolProvider = {
 			provideWorkspaceSymbols: (query, token) => {
-				return symbolConverter(locate.query(query));
+				return locate.query(query).then(symbolsConverter);
 			}
 		};
 		subs.push(vscode.languages.registerWorkspaceSymbolProvider(workspaceSymbolProvider));