From 266749b7b528dfbb5cdc51a995e27b0f61e6f12f Mon Sep 17 00:00:00 2001 From: phestermcs Date: Sat, 26 Nov 2016 06:56:33 -0700 Subject: [PATCH 1/4] module: add adjacent.node_modules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit (includes fix-preserve-symlinks) dependencies can be located like: /node_modules   /mod     /node_modules       /dep practically speaking, this prevents package managers from exploiting symlinks to a machine module store. but by adding and additional search path: /node_modules   /mod   /mod.node_modules     /dep modules can be symlinked to a machine store but still allow local trees to have dependency versions specific to the tree --- lib/module.js | 120 ++++++++++++++++++++++++++++++++----------- src/node.cc | 10 ++++ src/node_config.cc | 3 ++ src/node_internals.h | 1 + 4 files changed, 105 insertions(+), 29 deletions(-) diff --git a/lib/module.js b/lib/module.js index 09c4b85b9e5599..e5441db07aef0a 100644 --- a/lib/module.js +++ b/lib/module.js @@ -11,6 +11,7 @@ const path = require('path'); const internalModuleReadFile = process.binding('fs').internalModuleReadFile; const internalModuleStat = process.binding('fs').internalModuleStat; const preserveSymlinks = !!process.binding('config').preserveSymlinks; +const adjacentNodeModules = !!process.binding('config').adjacentNodeModules; // If obj.hasOwnProperty has been overridden, then calling // obj.hasOwnProperty(prop) will break. @@ -98,15 +99,15 @@ function readPackage(requestPath) { return pkg; } -function tryPackage(requestPath, exts, isMain) { +function tryPackage(requestPath, exts) { var pkg = readPackage(requestPath); if (!pkg) return false; var filename = path.resolve(requestPath, pkg); - return tryFile(filename, isMain) || - tryExtensions(filename, exts, isMain) || - tryExtensions(path.resolve(filename, 'index'), exts, isMain); + return tryFile(filename) || + tryExtensions(filename, exts) || + tryExtensions(path.resolve(filename, 'index'), exts); } // In order to minimize unnecessary lstat() calls, @@ -118,9 +119,9 @@ const realpathCache = new Map(); // if using --preserve-symlinks and isMain is false, // keep symlinks intact, otherwise resolve to the // absolute realpath. -function tryFile(requestPath, isMain) { +function tryFile(requestPath) { const rc = stat(requestPath); - if (preserveSymlinks && !isMain) { + if (preserveSymlinks) { return rc === 0 && path.resolve(requestPath); } return rc === 0 && toRealPath(requestPath); @@ -133,9 +134,9 @@ function toRealPath(requestPath) { } // given a path check a the file exists with any of the set extensions -function tryExtensions(p, exts, isMain) { +function tryExtensions(p, exts) { for (var i = 0; i < exts.length; i++) { - const filename = tryFile(p + exts[i], isMain); + const filename = tryFile(p + exts[i]); if (filename) { return filename; @@ -145,7 +146,7 @@ function tryExtensions(p, exts, isMain) { } var warned = false; -Module._findPath = function(request, paths, isMain) { +Module._findPath = function(request, paths) { if (path.isAbsolute(request)) { paths = ['']; } else if (!paths || paths.length === 0) { @@ -172,7 +173,7 @@ Module._findPath = function(request, paths, isMain) { const rc = stat(basePath); if (!trailingSlash) { if (rc === 0) { // File. - if (preserveSymlinks && !isMain) { + if (preserveSymlinks) { filename = path.resolve(basePath); } else { filename = toRealPath(basePath); @@ -180,28 +181,28 @@ Module._findPath = function(request, paths, isMain) { } else if (rc === 1) { // Directory. if (exts === undefined) exts = Object.keys(Module._extensions); - filename = tryPackage(basePath, exts, isMain); + filename = tryPackage(basePath, exts); } if (!filename) { // try it with each of the extensions if (exts === undefined) exts = Object.keys(Module._extensions); - filename = tryExtensions(basePath, exts, isMain); + filename = tryExtensions(basePath, exts); } } if (!filename && rc === 1) { // Directory. if (exts === undefined) exts = Object.keys(Module._extensions); - filename = tryPackage(basePath, exts, isMain); + filename = tryPackage(basePath, exts); } if (!filename && rc === 1) { // Directory. // try it with each of the extensions at "index" if (exts === undefined) exts = Object.keys(Module._extensions); - filename = tryExtensions(path.resolve(basePath, 'index'), exts, isMain); + filename = tryExtensions(path.resolve(basePath, 'index'), exts); } if (filename) { @@ -253,9 +254,22 @@ if (process.platform === 'win32') { // Use colon as an extra condition since we can get node_modules // path for dirver root like 'C:\node_modules' and don't need to // parse driver name. - if (code === 92/*\*/ || code === 47/*/*/ || code === 58/*:*/) { - if (p !== nmLen) - paths.push(from.slice(0, last) + '\\node_modules'); + if (code === 92/*\*/ || code === 47/*/*/ || code === 58/*:*/ + || (adjacentNodeModules && code === 46/*.*/)) { + if (p !== nmLen) { + var parent = from.slice(0, last); + paths.push(parent + '\\node_modules'); + + if (adjacentNodeModules) { + paths.push(parent + '.node_modules'); + if (code === 46/*.*/) + while (i > 1) { + const code = from.charCodeAt(--i); + if (code === 92/*\*/ || code === 47/*/*/ || code === 58/*:*/) + break; + } + } + } last = i; p = 0; } else if (p !== -1) { @@ -267,6 +281,9 @@ if (process.platform === 'win32') { } } + // superfluous "/.node_modules" + if (adjacentNodeModules) paths.pop(); + return paths; }; } else { // posix @@ -287,9 +304,17 @@ if (process.platform === 'win32') { var last = from.length; for (var i = from.length - 1; i >= 0; --i) { const code = from.charCodeAt(i); - if (code === 47/*/*/) { - if (p !== nmLen) - paths.push(from.slice(0, last) + '/node_modules'); + if (code === 47/*/*/ || (adjacentNodeModules && code === 46/*.*/)) { + if (p !== nmLen) { + var parent = from.slice(0, last); + paths.push(parent + '/node_modules'); + + if (adjacentNodeModules) { + paths.push(parent + '.node_modules'); + if (code === 46/*.*/) + while (i > 1 && from.charCodeAt(--i) !== 47/*/*/); + } + } last = i; p = 0; } else if (p !== -1) { @@ -402,6 +427,34 @@ Module._resolveLookupPaths = function(request, parent) { return [id, [path.dirname(parent.filename)]]; }; +// when preserving symlinks, if the main file is both a file symlink AND +// has execute permission (on windows, its extension is in PATHEXT), then +// its target is simply read and resolved, potentially relative to the +// symlink's dirname. in all other cases the original path is preserved. +Module._resolveMainRequest = function(request) { + if(!preserveSymlinks) return request + + request = Module._resolveFilename(request, null) + if(!fs.lstatSync(request).isSymbolicLink()) return request + + let isExecutable = false + if(process.platform === 'win32') { + const execExts = (process.env.PATHEXT || '').split(path.delimiter) + const requestExt = path.extname(request).toLowerCase() + isExecutable = + execExts.some(execExt => requestExt === execExt.trim().toLowerCase()) + } else try { + fs.accessSync(request, fs.constants.X_OK) + isExecutable = true + } catch(e) {} + + if(!isExecutable) return request + + const targetPath = fs.readlinkSync(request) + return path.isAbsolute(targetPath) + ? targetPath + : path.resolve(path.dirname(request), targetPath) +} // Check the cache for the requested file. // 1. If a module already exists in the cache: return its exports object. @@ -415,14 +468,23 @@ Module._load = function(request, parent, isMain) { debug('Module._load REQUEST %s parent: %s', request, parent.id); } - var filename = Module._resolveFilename(request, parent, isMain); + var requestPath = isMain ? Module._resolveMainRequest(request) : request; + var filename = Module._resolveFilename(requestPath, parent); + var doesNonInternalExist = NativeModule.nonInternalExists(filename); + + // when supporting symlinks, always cache by realpath to fix + // memory bloat and addon crashing. module's __dirname will be that + // of first requestPath that require()d it, which could be a symlink. + var cachePath = preserveSymlinks && !doesNonInternalExist + ? toRealPath(filename) + : filename; - var cachedModule = Module._cache[filename]; + var cachedModule = Module._cache[cachePath]; if (cachedModule) { return cachedModule.exports; } - if (NativeModule.nonInternalExists(filename)) { + if (doesNonInternalExist) { debug('load native module %s', request); return NativeModule.require(filename); } @@ -434,26 +496,26 @@ Module._load = function(request, parent, isMain) { module.id = '.'; } - Module._cache[filename] = module; + Module._cache[cachePath] = module; - tryModuleLoad(module, filename); + tryModuleLoad(module, filename, cachePath); return module.exports; }; -function tryModuleLoad(module, filename) { +function tryModuleLoad(module, filename, cachePath) { var threw = true; try { module.load(filename); threw = false; } finally { if (threw) { - delete Module._cache[filename]; + delete Module._cache[cachePath]; } } } -Module._resolveFilename = function(request, parent, isMain) { +Module._resolveFilename = function(request, parent) { if (NativeModule.nonInternalExists(request)) { return request; } @@ -465,7 +527,7 @@ Module._resolveFilename = function(request, parent, isMain) { // look up the filename first, since that's the cache key. debug('looking for %j in %j', id, paths); - var filename = Module._findPath(request, paths, isMain); + var filename = Module._findPath(request, paths); if (!filename) { var err = new Error("Cannot find module '" + request + "'"); err.code = 'MODULE_NOT_FOUND'; diff --git a/src/node.cc b/src/node.cc index e87bddcb6ff2cc..c827dd9b063ea1 100644 --- a/src/node.cc +++ b/src/node.cc @@ -186,6 +186,7 @@ bool trace_warnings = false; // Used in node_config.cc to set a constant on process.binding('config') // that is used by lib/module.js bool config_preserve_symlinks = false; +bool config_adjacent_node_modules = false; bool v8_initialized = false; @@ -4243,6 +4244,15 @@ void Init(int* argc, config_preserve_symlinks = (*preserve_symlinks == '1'); } + if (auto adjacent_nm = secure_getenv("NODE_ADJACENT_NODE_MODULES")) { + config_adjacent_node_modules = (*adjacent_nm == '1'); + } + if (auto support_symlinks = secure_getenv("NODE_SUPPORT_SYMLINKS")) { + config_preserve_symlinks = (*support_symlinks == '1'); + config_adjacent_node_modules = (*support_symlinks == '1'); + } + + // Parse a few arguments which are specific to Node. int v8_argc; const char** v8_argv; diff --git a/src/node_config.cc b/src/node_config.cc index 401345f6a608be..39cb1badee5c14 100644 --- a/src/node_config.cc +++ b/src/node_config.cc @@ -44,6 +44,9 @@ void InitConfig(Local target, if (config_preserve_symlinks) READONLY_BOOLEAN_PROPERTY("preserveSymlinks"); + + if (config_adjacent_node_modules) + READONLY_BOOLEAN_PROPERTY("adjacentNodeModules"); } // InitConfig } // namespace node diff --git a/src/node_internals.h b/src/node_internals.h index 0a65be7642ff2a..de355d9ababf43 100644 --- a/src/node_internals.h +++ b/src/node_internals.h @@ -40,6 +40,7 @@ extern const char* openssl_config; // Used in node_config.cc to set a constant on process.binding('config') // that is used by lib/module.js extern bool config_preserve_symlinks; +extern bool config_adjacent_node_modules; // Tells whether it is safe to call v8::Isolate::GetCurrent(). extern bool v8_initialized; From d0306de33f8645e4bd1670a28957c48db36fb370 Mon Sep 17 00:00:00 2001 From: phestermcs Date: Mon, 5 Dec 2016 17:28:04 -0700 Subject: [PATCH 2/4] module: '.node_modules' to '+node_modules' change path character for adjacent-node-modules directory from '.' to '+' to ensure directory names can never clash with module names --- lib/module.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/module.js b/lib/module.js index e5441db07aef0a..7464301fbf4ee3 100644 --- a/lib/module.js +++ b/lib/module.js @@ -261,7 +261,7 @@ if (process.platform === 'win32') { paths.push(parent + '\\node_modules'); if (adjacentNodeModules) { - paths.push(parent + '.node_modules'); + paths.push(parent + '+node_modules'); if (code === 46/*.*/) while (i > 1) { const code = from.charCodeAt(--i); @@ -310,7 +310,7 @@ if (process.platform === 'win32') { paths.push(parent + '/node_modules'); if (adjacentNodeModules) { - paths.push(parent + '.node_modules'); + paths.push(parent + '+node_modules'); if (code === 46/*.*/) while (i > 1 && from.charCodeAt(--i) !== 47/*/*/); } From 51757d355a07fa431d54c908db992ff650e3f70a Mon Sep 17 00:00:00 2001 From: phestermcs Date: Mon, 5 Dec 2016 18:22:02 -0700 Subject: [PATCH 3/4] module: apply code style changes --- lib/module.js | 55 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/lib/module.js b/lib/module.js index 7464301fbf4ee3..15cd9dc5086bb9 100644 --- a/lib/module.js +++ b/lib/module.js @@ -254,20 +254,22 @@ if (process.platform === 'win32') { // Use colon as an extra condition since we can get node_modules // path for dirver root like 'C:\node_modules' and don't need to // parse driver name. - if (code === 92/*\*/ || code === 47/*/*/ || code === 58/*:*/ - || (adjacentNodeModules && code === 46/*.*/)) { + if (code === 92/*\*/ || code === 47/*/*/ || code === 58/*:*/ || + (adjacentNodeModules && code === 46/*.*/)) { if (p !== nmLen) { var parent = from.slice(0, last); paths.push(parent + '\\node_modules'); if (adjacentNodeModules) { paths.push(parent + '+node_modules'); - if (code === 46/*.*/) + if (code === 46/*.*/) { while (i > 1) { const code = from.charCodeAt(--i); - if (code === 92/*\*/ || code === 47/*/*/ || code === 58/*:*/) + if (code === 92/*\*/ || code === 47/*/*/ || code === 58/*:*/) { break; + } } + } } } last = i; @@ -311,8 +313,13 @@ if (process.platform === 'win32') { if (adjacentNodeModules) { paths.push(parent + '+node_modules'); - if (code === 46/*.*/) - while (i > 1 && from.charCodeAt(--i) !== 47/*/*/); + if (code === 46/*.*/) { + while (i > 1) { + if (from.charCodeAt(--i) === 47/*/*/) { + break; + } + } + } } } last = i; @@ -432,28 +439,36 @@ Module._resolveLookupPaths = function(request, parent) { // its target is simply read and resolved, potentially relative to the // symlink's dirname. in all other cases the original path is preserved. Module._resolveMainRequest = function(request) { - if(!preserveSymlinks) return request + if(!preserveSymlinks) { + return request; + } - request = Module._resolveFilename(request, null) - if(!fs.lstatSync(request).isSymbolicLink()) return request + request = Module._resolveFilename(request, null); + if(!fs.lstatSync(request).isSymbolicLink()) { + return request; + } - let isExecutable = false + let isExecutable = false; if(process.platform === 'win32') { - const execExts = (process.env.PATHEXT || '').split(path.delimiter) - const requestExt = path.extname(request).toLowerCase() + const execExts = (process.env.PATHEXT || '').split(path.delimiter); + const requestExt = path.extname(request).toLowerCase(); isExecutable = - execExts.some(execExt => requestExt === execExt.trim().toLowerCase()) - } else try { - fs.accessSync(request, fs.constants.X_OK) - isExecutable = true - } catch(e) {} + execExts.some((execExt) => requestExt === execExt.trim().toLowerCase()); + } else { + try { + fs.accessSync(request, fs.constants.X_OK); + isExecutable = true; + } catch(e) {} + } - if(!isExecutable) return request + if(!isExecutable) { + return request; + } - const targetPath = fs.readlinkSync(request) + const targetPath = fs.readlinkSync(request); return path.isAbsolute(targetPath) ? targetPath - : path.resolve(path.dirname(request), targetPath) + : path.resolve(path.dirname(request), targetPath); } // Check the cache for the requested file. From 3b7fca6f9a7140cae628514cb004a6605b450108 Mon Sep 17 00:00:00 2001 From: phestermcs Date: Tue, 6 Dec 2016 13:50:29 -0700 Subject: [PATCH 4/4] module: simplify _resolveMain remove check of execute bit for file-symlink as it's always set. if file-symlink, just follow its target --- lib/module.js | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) diff --git a/lib/module.js b/lib/module.js index 15cd9dc5086bb9..a46adcb98ccbd1 100644 --- a/lib/module.js +++ b/lib/module.js @@ -255,14 +255,14 @@ if (process.platform === 'win32') { // path for dirver root like 'C:\node_modules' and don't need to // parse driver name. if (code === 92/*\*/ || code === 47/*/*/ || code === 58/*:*/ || - (adjacentNodeModules && code === 46/*.*/)) { + (adjacentNodeModules && code === 43/*+*/)) { if (p !== nmLen) { var parent = from.slice(0, last); paths.push(parent + '\\node_modules'); if (adjacentNodeModules) { paths.push(parent + '+node_modules'); - if (code === 46/*.*/) { + if (code === 43/*+*/) { while (i > 1) { const code = from.charCodeAt(--i); if (code === 92/*\*/ || code === 47/*/*/ || code === 58/*:*/) { @@ -306,14 +306,14 @@ if (process.platform === 'win32') { var last = from.length; for (var i = from.length - 1; i >= 0; --i) { const code = from.charCodeAt(i); - if (code === 47/*/*/ || (adjacentNodeModules && code === 46/*.*/)) { + if (code === 47/*/*/ || (adjacentNodeModules && code === 43/*+*/)) { if (p !== nmLen) { var parent = from.slice(0, last); paths.push(parent + '/node_modules'); if (adjacentNodeModules) { paths.push(parent + '+node_modules'); - if (code === 46/*.*/) { + if (code === 43/*+*/) { while (i > 1) { if (from.charCodeAt(--i) === 47/*/*/) { break; @@ -434,8 +434,7 @@ Module._resolveLookupPaths = function(request, parent) { return [id, [path.dirname(parent.filename)]]; }; -// when preserving symlinks, if the main file is both a file symlink AND -// has execute permission (on windows, its extension is in PATHEXT), then +// when preserving symlinks, if the main file is a file symlink then // its target is simply read and resolved, potentially relative to the // symlink's dirname. in all other cases the original path is preserved. Module._resolveMainRequest = function(request) { @@ -448,23 +447,6 @@ Module._resolveMainRequest = function(request) { return request; } - let isExecutable = false; - if(process.platform === 'win32') { - const execExts = (process.env.PATHEXT || '').split(path.delimiter); - const requestExt = path.extname(request).toLowerCase(); - isExecutable = - execExts.some((execExt) => requestExt === execExt.trim().toLowerCase()); - } else { - try { - fs.accessSync(request, fs.constants.X_OK); - isExecutable = true; - } catch(e) {} - } - - if(!isExecutable) { - return request; - } - const targetPath = fs.readlinkSync(request); return path.isAbsolute(targetPath) ? targetPath