diff --git a/DEPENDENCIES.md b/DEPENDENCIES.md index dec82172702ab..2c159a4bb37ef 100644 --- a/DEPENDENCIES.md +++ b/DEPENDENCIES.md @@ -601,6 +601,7 @@ graph LR; npmcli-arborist-->proggy; npmcli-arborist-->promise-all-reject-late; npmcli-arborist-->promise-call-limit; + npmcli-arborist-->promise-retry; npmcli-arborist-->read-package-json-fast; npmcli-arborist-->semver; npmcli-arborist-->ssri; diff --git a/package-lock.json b/package-lock.json index 141373e1f17b9..202e996d8ac4a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17922,6 +17922,7 @@ "proggy": "^3.0.0", "promise-all-reject-late": "^1.0.0", "promise-call-limit": "^3.0.1", + "promise-retry": "^2.0.1", "read-package-json-fast": "^4.0.0", "semver": "^7.3.7", "ssri": "^12.0.0", diff --git a/workspaces/arborist/lib/arborist/isolated-reifier.js b/workspaces/arborist/lib/arborist/isolated-reifier.js index 4fbcd801fdf63..175c945ab95ae 100644 --- a/workspaces/arborist/lib/arborist/isolated-reifier.js +++ b/workspaces/arborist/lib/arborist/isolated-reifier.js @@ -1,72 +1,105 @@ -const _makeIdealGraph = Symbol('makeIdealGraph') -const _createIsolatedTree = Symbol.for('createIsolatedTree') -const _createBundledTree = Symbol('createBundledTree') const { mkdirSync } = require('node:fs') const pacote = require('pacote') const { join } = require('node:path') const { depth } = require('treeverse') const crypto = require('node:crypto') -// cache complicated function results -const memoize = (fn) => { - const memo = new Map() - return async function (arg) { - const key = arg - if (memo.has(key)) { - return memo.get(key) - } - const result = {} - memo.set(key, result) - await fn(result, arg) - return result - } +// generate short hash key based on the dependency tree starting at this node +const getKey = (startNode) => { + const deps = [] + const branch = [] + depth({ + tree: startNode, + getChildren: node => node.dependencies, + visit: node => { + branch.push(`${node.packageName}@${node.version}`) + deps.push(`${branch.join('->')}::${node.resolved}`) + }, + leave: () => { + branch.pop() + }, + }) + deps.sort() + // TODO these replaces were originally to deal with node 14 not supporting base64url and likely don't need to happen anymore + // Changing this is a pretty significant breaking change, but removing parts of the hash increases collision possibilities (even if slight). + const hash = crypto.createHash('shake256', { outputLength: 16 }) + .update(deps.join(',')) + .digest('base64') + .replace(/\+/g, '-') + .replace(/\//g, '_') + .replace(/=+$/m, '') + return `${startNode.packageName}@${startNode.version}-${hash}` } module.exports = cls => class IsolatedReifier extends cls { + #externalProxies = new Map() + #processedEdges = new Set() + #workspaceProxies = new Map() + + #generateChild (node, location, pkg, inStore, root) { + const newChild = { + binPaths: [], + children: new Map(), + edgesIn: new Set(), + edgesOut: new Map(), + fsChildren: new Set(), + /* istanbul ignore next -- emulate Node */ + getBundler () { + return null + }, + global: false, + globalTop: false, + hasShrinkwrap: false, + inDepBundle: false, + integrity: null, + isInStore: inStore, + isLink: false, + isProjectRoot: false, + isRoot: false, + isTop: false, + location, + name: node.packageName || node.name, + optional: node.optional, + package: pkg, + parent: root, + path: join(this.idealGraph.root.localPath, location), + realpath: join(this.idealGraph.root.localPath, location), + resolved: node.resolved, + root, + top: { path: this.idealGraph.root.localPath }, + version: pkg.version, + } + newChild.target = newChild + root.children.set(newChild.location, newChild) + root.inventory.set(newChild.location, newChild) + } + /** * Create an ideal graph. * * An implementation of npm RFC-0042 * https://github.com/npm/rfcs/blob/main/accepted/0042-isolated-mode.md * - * This entire file should be considered technical debt that will be resolved - * with an Arborist refactor or rewrite. Embedded logic in Nodes and Links, - * and the incremental state of building trees and reifying contains too many - * assumptions to do a linked mode properly. + * This entire file should be considered technical debt that will be resolved with an Arborist refactor or rewrite. + * Embedded logic in Nodes and Links, and the incremental state of building trees and reifying contains too many assumptions to do a linked mode properly. * - * Instead, this approach takes a tree built from build-ideal-tree, and - * returns a new tree-like structure without the embedded logic of Node and - * Link classes. + * Instead, this approach takes a tree built from build-ideal-tree, and returns a new tree-like structure without the embedded logic of Node and Link classes. * - * Since the RFC requires leaving the package-lock in place, this approach - * temporarily replaces the tree state for a couple of steps of reifying. + * Since the RFC requires leaving the package-lock in place, this approach temporarily replaces the tree state for a couple of steps of reifying. * **/ - async [_makeIdealGraph] (options) { - /* Make sure that the ideal tree is build as the rest of - * the algorithm depends on it. - */ - const bitOpt = { - ...options, - complete: false, - } - await this.buildIdealTree(bitOpt) + async makeIdealGraph () { const idealTree = this.idealTree - this.rootNode = {} - const root = this.rootNode + this.idealGraph = { + external: [], + isProjectRoot: true, + localLocation: idealTree.location, + localPath: idealTree.path, + } this.counter = 0 - // memoize to cache generating proxy Nodes - this.externalProxyMemo = memoize(this.externalProxy.bind(this)) - this.workspaceProxyMemo = memoize(this.workspaceProxy.bind(this)) - - root.external = [] - root.isProjectRoot = true - root.localLocation = idealTree.location - root.localPath = idealTree.path - root.workspaces = await Promise.all( - Array.from(idealTree.fsChildren.values(), this.workspaceProxyMemo)) + this.idealGraph.workspaces = await Promise.all(Array.from(idealTree.fsChildren.values(), w => this.workspaceProxy(w))) const processed = new Set() const queue = [idealTree, ...idealTree.fsChildren] while (queue.length !== 0) { @@ -75,42 +108,53 @@ module.exports = cls => class IsolatedReifier extends cls { continue } processed.add(next.location) - next.edgesOut.forEach(e => { - if (!e.to || (next.package.bundleDependencies || next.package.bundledDependencies || []).includes(e.to.name)) { - return + next.edgesOut.forEach(edge => { + if (edge.to && !(next.package.bundleDependencies || next.package.bundledDependencies || []).includes(edge.to.name)) { + queue.push(edge.to) } - queue.push(e.to) }) - if (!next.isProjectRoot && !next.isWorkspace) { - root.external.push(await this.externalProxyMemo(next)) + // local `file:` deps are in fsChildren but are not workspaces. + // they are already handled as workspace-like proxies above and should not go through the external/store extraction path. + if (!next.isProjectRoot && !next.isWorkspace && !next.inert && !idealTree.fsChildren.has(next) && !idealTree.fsChildren.has(next.target)) { + this.idealGraph.external.push(await this.externalProxy(next)) } } - await this.assignCommonProperties(idealTree, root) - - this.idealGraph = root + await this.assignCommonProperties(idealTree, this.idealGraph) } - async workspaceProxy (result, node) { + async workspaceProxy (node) { + if (this.#workspaceProxies.has(node)) { + return this.#workspaceProxies.get(node) + } + const result = {} + // XXX this goes recursive if we don't set here because assignCommonProperties also calls this.workspaceProxy + this.#workspaceProxies.set(node, result) result.localLocation = node.location result.localPath = node.path result.isWorkspace = true result.resolved = node.resolved await this.assignCommonProperties(node, result) + return result } - async externalProxy (result, node) { - await this.assignCommonProperties(node, result) + async externalProxy (node) { + if (this.#externalProxies.has(node)) { + return this.#externalProxies.get(node) + } + const result = {} + // XXX this goes recursive if we don't set here because assignCommonProperties also calls this.externalProxy + this.#externalProxies.set(node, result) + await this.assignCommonProperties(node, result, !node.hasShrinkwrap) if (node.hasShrinkwrap) { const dir = join( node.root.path, 'node_modules', '.store', - `${node.name}@${node.version}` + `${node.packageName}@${node.version}` ) mkdirSync(dir, { recursive: true }) - // TODO this approach feels wrong - // and shouldn't be necessary for shrinkwraps + // TODO this approach feels wrong and shouldn't be necessary for shrinkwraps await pacote.extract(node.resolved, dir, { ...this.options, resolved: node.resolved, @@ -118,51 +162,81 @@ module.exports = cls => class IsolatedReifier extends cls { }) const Arborist = this.constructor const arb = new Arborist({ ...this.options, path: dir }) - await arb[_makeIdealGraph]({ dev: false }) - this.rootNode.external.push(...arb.idealGraph.external) - arb.idealGraph.external.forEach(e => { - e.root = this.rootNode - e.id = `${node.id}=>${e.id}` + // Make sure that the ideal tree is build as the rest of the algorithm depends on it. + await arb.buildIdealTree({ + complete: false, + dev: false, }) + await arb.makeIdealGraph() + this.idealGraph.external.push(...arb.idealGraph.external) + for (const edge of arb.idealGraph.external) { + edge.root = this.idealGraph + edge.id = `${node.id}=>${edge.id}` + } result.localDependencies = [] result.externalDependencies = arb.idealGraph.externalDependencies result.externalOptionalDependencies = arb.idealGraph.externalOptionalDependencies result.dependencies = [ ...result.externalDependencies, - ...result.localDependencies, ...result.externalOptionalDependencies, ] } result.optional = node.optional result.resolved = node.resolved result.version = node.version + return result } - async assignCommonProperties (node, result) { - function validEdgesOut (node) { - return [...node.edgesOut.values()].filter(e => e.to && e.to.target && !(node.package.bundledDepenedencies || node.package.bundleDependencies || []).includes(e.to.name)) + async assignCommonProperties (node, result, populateDeps = true) { + result.root = this.idealGraph + result.id = this.counter++ + /* istanbul ignore next - packageName is always set for real packages */ + result.name = result.isWorkspace ? (node.packageName || node.name) : node.name + /* istanbul ignore next - packageName is always set for real packages */ + result.packageName = node.packageName || node.name + result.package = { ...node.package } + result.package.bundleDependencies = undefined + result.hasInstallScript = node.hasInstallScript + + if (!populateDeps) { + return + } + + const edges = [...node.edgesOut.values()].filter(edge => + edge.to?.target && + !(node.package.bundledDependencies || node.package.bundleDependencies)?.includes(edge.to.name) + ) + const nonOptionalDeps = edges.filter(edge => !edge.optional).map(edge => edge.to.target) + + // When legacyPeerDeps is enabled, peer dep edges are not created on the node. + // Resolve them from the tree so they get symlinked in the store. + const peerDeps = node.package.peerDependencies + if (peerDeps && node.legacyPeerDeps) { + const edgeNames = new Set(edges.map(edge => edge.name)) + for (const peerName in peerDeps) { + if (!edgeNames.has(peerName)) { + const resolved = node.resolve(peerName) + if (resolved && resolved !== node && !resolved.inert) { + nonOptionalDeps.push(resolved.target) + } + } + } } - const edges = validEdgesOut(node) - const optionalDeps = edges.filter(e => e.optional).map(e => e.to.target) - const nonOptionalDeps = edges.filter(e => !e.optional).map(e => e.to.target) - result.localDependencies = await Promise.all(nonOptionalDeps.filter(n => n.isWorkspace).map(this.workspaceProxyMemo)) - result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !n.isWorkspace).map(this.externalProxyMemo)) - result.externalOptionalDependencies = await Promise.all(optionalDeps.map(this.externalProxyMemo)) + // local `file:` deps (non-workspace fsChildren) should be treated as local dependencies, not external, so they get symlinked directly instead of being extracted into the store. + const isLocal = (n) => n.isWorkspace || node.fsChildren?.has(n) + const optionalDeps = edges.filter(edge => edge.optional).map(edge => edge.to.target) + result.localDependencies = await Promise.all(nonOptionalDeps.filter(isLocal).map(n => this.workspaceProxy(n))) + result.externalDependencies = await Promise.all(nonOptionalDeps.filter(n => !isLocal(n) && !n.inert).map(n => this.externalProxy(n))) + result.externalOptionalDependencies = await Promise.all(optionalDeps.filter(n => !n.inert).map(n => this.externalProxy(n))) result.dependencies = [ ...result.externalDependencies, ...result.localDependencies, ...result.externalOptionalDependencies, ] - result.root = this.rootNode - result.id = this.counter++ - result.name = node.name - result.package = { ...node.package } - result.package.bundleDependencies = undefined - result.hasInstallScript = node.hasInstallScript } - async [_createBundledTree] () { + async #createBundledTree () { // TODO: make sure that idealTree object exists const idealTree = this.idealTree // TODO: test workspaces having bundled deps @@ -201,253 +275,232 @@ module.exports = cls => class IsolatedReifier extends cls { nodes.set(to.location, { location: to.location, resolved: to.resolved, name: to.name, optional: to.optional, pkg: { ...to.package, bundleDependencies: undefined } }) edges.push({ from: from.isRoot ? 'root' : from.location, to: to.location }) - to.edgesOut.forEach(e => { + to.edgesOut.forEach(edge => { // an edge out should always have a to /* istanbul ignore else */ - if (e.to) { - queue.push({ from: e.from, to: e.to }) + if (edge.to) { + queue.push({ from: edge.from, to: edge.to }) } }) } return { edges, nodes } } - async [_createIsolatedTree] () { - await this[_makeIdealGraph](this.options) - - const proxiedIdealTree = this.idealGraph - - const bundledTree = await this[_createBundledTree]() - - const treeHash = (startNode) => { - // generate short hash based on the dependency tree - // starting at this node - const deps = [] - const branch = [] - depth({ - tree: startNode, - getChildren: node => node.dependencies, - filter: node => node, - visit: node => { - branch.push(`${node.name}@${node.version}`) - deps.push(`${branch.join('->')}::${node.resolved}`) - }, - leave: () => { - branch.pop() - }, - }) - deps.sort() - return crypto.createHash('shake256', { outputLength: 16 }) - .update(deps.join(',')) - .digest('base64') - // Node v14 doesn't support base64url - .replace(/\+/g, '-') - .replace(/\//g, '_') - .replace(/=+$/m, '') - } + async createIsolatedTree () { + await this.makeIdealGraph() - const getKey = (idealTreeNode) => { - return `${idealTreeNode.name}@${idealTreeNode.version}-${treeHash(idealTreeNode)}` - } + const bundledTree = await this.#createBundledTree() const root = { - fsChildren: [], - integrity: null, - inventory: new Map(), - isLink: false, - isRoot: true, binPaths: [], + children: new Map(), edgesIn: new Set(), edgesOut: new Map(), + fsChildren: new Set(), + global: false, hasShrinkwrap: false, + integrity: null, + inventory: new Map(), + isLink: false, + isProjectRoot: true, + isRoot: true, + isTop: true, + linksIn: new Set(), + meta: { loadedFromDisk: false }, + package: this.idealGraph.root.package, parent: null, + path: this.idealGraph.root.localPath, + realpath: this.idealGraph.root.localPath, // TODO: we should probably not reference this.idealTree resolved: this.idealTree.resolved, - isTop: true, - path: proxiedIdealTree.root.localPath, - realpath: proxiedIdealTree.root.localPath, - package: proxiedIdealTree.root.package, - meta: { loadedFromDisk: false }, - global: false, - isProjectRoot: true, - children: [], + tops: new Set(), + workspaces: new Map(), } - // root.inventory.set('', t) - // root.meta = this.idealTree.meta - // TODO We should mock better the inventory object because it is used by audit-report.js ... maybe + root.inventory.set('', root) + root.root = root + root.target = root + // TODO inventory.query is a stub; audit-report needs 'packageName' support root.inventory.query = () => { return [] } const processed = new Set() - proxiedIdealTree.workspaces.forEach(c => { + for (const c of this.idealGraph.workspaces) { + const wsName = c.packageName const workspace = { + binPaths: [], + children: new Map(), edgesIn: new Set(), edgesOut: new Map(), - children: [], + fsChildren: new Set(), hasInstallScript: c.hasInstallScript, - binPaths: [], - package: c.package, + isLink: false, + isRoot: false, + linksIn: new Set(), location: c.localLocation, + name: wsName, + package: c.package, path: c.localPath, realpath: c.localPath, resolved: c.resolved, } - root.fsChildren.push(workspace) + workspace.target = workspace + root.fsChildren.add(workspace) root.inventory.set(workspace.location, workspace) - }) - const generateChild = (node, location, pkg, inStore) => { - const newChild = { + + // Create workspace Link entry in children for _diffTrees lookup + const wsLink = { + binPaths: [], + children: new Map(), + edgesIn: new Set(), + edgesOut: new Map(), + fsChildren: new Set(), global: false, globalTop: false, + isLink: true, isProjectRoot: false, - isTop: false, - location, - name: node.name, - optional: node.optional, - top: { path: proxiedIdealTree.root.localPath }, - children: [], - edgesIn: new Set(), - edgesOut: new Map(), - binPaths: [], - fsChildren: [], - /* istanbul ignore next -- emulate Node */ - getBundler () { - return null - }, - hasShrinkwrap: false, - inDepBundle: false, - integrity: null, - isLink: false, isRoot: false, - isInStore: inStore, - path: join(proxiedIdealTree.root.localPath, location), - realpath: join(proxiedIdealTree.root.localPath, location), - resolved: node.resolved, - version: pkg.version, - package: pkg, + isTop: false, + linksIn: new Set(), + location: join('node_modules', wsName), + name: wsName, + package: workspace.package, + parent: root, + path: join(root.path, 'node_modules', wsName), + realpath: workspace.path, + root, + target: workspace, } - newChild.target = newChild - root.children.push(newChild) - root.inventory.set(newChild.location, newChild) + root.children.set(wsLink.name, wsLink) + root.inventory.set(wsLink.location, wsLink) + root.workspaces.set(wsName, workspace.path) + workspace.linksIn.add(wsLink) } - proxiedIdealTree.external.forEach(c => { + + this.idealGraph.external.forEach(c => { const key = getKey(c) if (processed.has(key)) { return } processed.add(key) - const location = join('node_modules', '.store', key, 'node_modules', c.name) - generateChild(c, location, c.package, true) + const location = join('node_modules', '.store', key, 'node_modules', c.packageName) + this.#generateChild(c, location, c.package, true, root) }) + bundledTree.nodes.forEach(node => { - generateChild(node, node.location, node.pkg, false) + this.#generateChild(node, node.location, node.pkg, false, root) }) - bundledTree.edges.forEach(e => { - const from = e.from === 'root' ? root : root.inventory.get(e.from) - const to = root.inventory.get(e.to) + + bundledTree.edges.forEach(edge => { + const from = edge.from === 'root' ? root : root.inventory.get(edge.from) + const to = root.inventory.get(edge.to) // Maybe optional should be propagated from the original edge - const edge = { optional: false, from, to } - from.edgesOut.set(to.name, edge) - to.edgesIn.add(edge) + const newEdge = { optional: false, from, to } + from.edgesOut.set(to.name, newEdge) + to.edgesIn.add(newEdge) }) - const memo = new Set() - function processEdges (node, externalEdge) { - externalEdge = !!externalEdge - const key = getKey(node) - if (memo.has(key)) { - return - } - memo.add(key) - - let from, nmFolder - if (externalEdge) { - const fromLocation = join('node_modules', '.store', key, 'node_modules', node.name) - from = root.children.find(c => c.location === fromLocation) - nmFolder = join('node_modules', '.store', key, 'node_modules') - } else { - from = node.isProjectRoot ? root : root.fsChildren.find(c => c.location === node.localLocation) - nmFolder = join(node.localLocation, 'node_modules') - } + this.#processEdges(this.idealGraph, false, root) + for (const node of this.idealGraph.workspaces) { + this.#processEdges(node, false, root) + } + return root + } - const processDeps = (dep, optional, external) => { - optional = !!optional - external = !!external + #processEdges (node, externalEdge, root) { + const key = getKey(node) + if (this.#processedEdges.has(key)) { + return + } + this.#processedEdges.add(key) - const location = join(nmFolder, dep.name) - const binNames = dep.package.bin && Object.keys(dep.package.bin) || [] - const toKey = getKey(dep) + let from, nmFolder + if (externalEdge) { + const fromLocation = join('node_modules', '.store', key, 'node_modules', node.packageName) + from = root.children.get(fromLocation) + nmFolder = join('node_modules', '.store', key, 'node_modules') + } else { + from = node.isProjectRoot ? root : root.inventory.get(node.localLocation) + nmFolder = join(node.localLocation, 'node_modules') + } + /* istanbul ignore next - strict-peer-deps can exclude nodes from the tree */ + if (!from) { + return + } - let target - if (external) { - const toLocation = join('node_modules', '.store', toKey, 'node_modules', dep.name) - target = root.children.find(c => c.location === toLocation) - } else { - target = root.fsChildren.find(c => c.location === dep.localLocation) - } - // TODO: we should no-op is an edge has already been created with the same fromKey and toKey - - binNames.forEach(bn => { - target.binPaths.push(join(from.realpath, 'node_modules', '.bin', bn)) - }) - - const link = { - global: false, - globalTop: false, - isProjectRoot: false, - edgesIn: new Set(), - edgesOut: new Map(), - binPaths: [], - isTop: false, - optional, - location: location, - path: join(dep.root.localPath, nmFolder, dep.name), - realpath: target.path, - name: toKey, - resolved: dep.resolved, - top: { path: dep.root.localPath }, - children: [], - fsChildren: [], - isLink: true, - isStoreLink: true, - isRoot: false, - package: { _id: 'abc', bundleDependencies: undefined, deprecated: undefined, bin: target.package.bin, scripts: dep.package.scripts }, - target, - } - const newEdge1 = { optional, from, to: link } - from.edgesOut.set(dep.name, newEdge1) - link.edgesIn.add(newEdge1) - const newEdge2 = { optional: false, from: link, to: target } - link.edgesOut.set(dep.name, newEdge2) - target.edgesIn.add(newEdge2) - root.children.push(link) - } + for (const dep of node.localDependencies) { + this.#processEdges(dep, false, root) + // nonOptional, local + this.#processDeps(dep, false, false, root, from, nmFolder) + } + for (const dep of node.externalDependencies) { + this.#processEdges(dep, true, root) + // nonOptional, external + this.#processDeps(dep, false, true, root, from, nmFolder) + } + for (const dep of node.externalOptionalDependencies) { + this.#processEdges(dep, true, root) + // optional, external + this.#processDeps(dep, true, true, root, from, nmFolder) + } + } - for (const dep of node.localDependencies) { - processEdges(dep, false) - // nonOptional, local - processDeps(dep, false, false) - } - for (const dep of node.externalDependencies) { - processEdges(dep, true) - // nonOptional, external - processDeps(dep, false, true) - } - for (const dep of node.externalOptionalDependencies) { - processEdges(dep, true) - // optional, external - processDeps(dep, true, true) + #processDeps (dep, optional, external, root, from, nmFolder) { + const toKey = getKey(dep) + + let target + if (external) { + const toLocation = join('node_modules', '.store', toKey, 'node_modules', dep.packageName) + target = root.children.get(toLocation) + } else { + target = root.inventory.get(dep.localLocation) + } + // TODO: we should no-op is an edge has already been created with the same fromKey and toKey + /* istanbul ignore next - strict-peer-deps can exclude nodes from the tree */ + if (!target) { + return + } + + if (dep.package.bin) { + for (const bn in dep.package.bin) { + target.binPaths.push(join(dep.root.localPath, nmFolder, '.bin', bn)) } } - processEdges(proxiedIdealTree, false) - for (const node of proxiedIdealTree.workspaces) { - processEdges(node, false) + const link = { + binPaths: [], + children: new Map(), + edgesIn: new Set(), + edgesOut: new Map(), + fsChildren: new Set(), + global: false, + globalTop: false, + isLink: true, + isProjectRoot: false, + isRoot: false, + isStoreLink: true, + isTop: false, + location: join(nmFolder, dep.name), + name: toKey, + optional, + // TODO _id: 'abc' ? + package: { _id: 'abc', bundleDependencies: undefined, deprecated: undefined, bin: target.package.bin, scripts: dep.package.scripts }, + parent: root, + path: join(dep.root.localPath, nmFolder, dep.name), + realpath: target.path, + resolved: external + ? `file:.store/${toKey}/node_modules/${dep.packageName}` + : dep.resolved, + root, + target, + version: dep.version, + top: { path: dep.root.localPath }, } - root.children.forEach(c => c.parent = root) - root.children.forEach(c => c.root = root) - root.root = root - root.target = root - return root + const newEdge1 = { optional, from, to: link } + from.edgesOut.set(dep.name, newEdge1) + link.edgesIn.add(newEdge1) + const newEdge2 = { optional: false, from: link, to: target } + link.edgesOut.set(dep.name, newEdge2) + target.edgesIn.add(newEdge2) + root.children.set(link.location, link) } } diff --git a/workspaces/arborist/lib/arborist/rebuild.js b/workspaces/arborist/lib/arborist/rebuild.js index 82f84772f9a85..240d9d258eaf1 100644 --- a/workspaces/arborist/lib/arborist/rebuild.js +++ b/workspaces/arborist/lib/arborist/rebuild.js @@ -297,12 +297,12 @@ module.exports = cls => class Builder extends cls { devOptional, package: pkg, location, - isStoreLink, } = node.target // skip any that we know we'll be deleting - // or storeLinks - if (this[_trashList].has(path) || isStoreLink) { + // or links to store entries (their scripts run on the store + // entry itself, not through the link) + if (this[_trashList].has(path) || (node.isLink && node.target?.isInStore)) { return } @@ -383,13 +383,21 @@ module.exports = cls => class Builder extends cls { const timeEnd = time.start(`build:link:${node.location}`) - const p = binLinks({ + // On Windows, antivirus/indexer can transiently lock files, causing EPERM/EACCES/EBUSY on the rename inside write-file-atomic (used by bin-links/fix-bin.js), so, retry with backoff. + const promiseRetry = require('promise-retry') + const p = promiseRetry((retry) => binLinks({ pkg: node.package, path: node.path, top: !!(node.isTop || node.globalTop), force: this.options.force, global: !!node.globalTop, - }) + }).catch(/* istanbul ignore next - Windows-only transient antivirus locks */ err => { + if (process.platform === 'win32' && + (err.code === 'EPERM' || err.code === 'EACCES' || err.code === 'EBUSY')) { + return retry(err) + } + throw err + }), { retries: 5, minTimeout: 500 }) await (this.#doHandleOptionalFailure ? this[_handleOptionalFailure](node, p) diff --git a/workspaces/arborist/lib/arborist/reify.js b/workspaces/arborist/lib/arborist/reify.js index 4083d79f4fa25..819df52eb823c 100644 --- a/workspaces/arborist/lib/arborist/reify.js +++ b/workspaces/arborist/lib/arborist/reify.js @@ -11,11 +11,13 @@ const { log, time } = require('proc-log') const hgi = require('hosted-git-info') const rpj = require('read-package-json-fast') -const { dirname, resolve, relative, join } = require('node:path') +const { dirname, resolve, relative, join, sep } = require('node:path') const { depth: dfwalk } = require('treeverse') +const { existsSync } = require('node:fs') const { lstat, mkdir, + readdir, rm, symlink, } = require('node:fs/promises') @@ -77,8 +79,6 @@ const _usePackageLock = Symbol.for('usePackageLock') // used by build-ideal-tree mixin const _addNodeToTrashList = Symbol.for('addNodeToTrashList') -const _createIsolatedTree = Symbol.for('createIsolatedTree') - module.exports = cls => class Reifier extends cls { #bundleMissing = new Set() // child nodes we'd EXPECT to be included in a bundle, but aren't #bundleUnpacked = new Set() // the nodes we unpack to read their bundles @@ -93,6 +93,7 @@ module.exports = cls => class Reifier extends cls { #shrinkwrapInflated = new Set() #sparseTreeDirs = new Set() #sparseTreeRoots = new Set() + #linkedActualForDiff = null constructor (options) { super(options) @@ -136,16 +137,21 @@ module.exports = cls => class Reifier extends cls { // this is currently technical debt which will be resolved in a refactor // of Node/Link trees log.warn('reify', 'The "linked" install strategy is EXPERIMENTAL and may contain bugs.') - this.idealTree = await this[_createIsolatedTree]() + this.idealTree = await this.createIsolatedTree() + this.#linkedActualForDiff = this.#buildLinkedActualForDiff( + this.idealTree, this.actualTree + ) } await this[_diffTrees]() await this[_reifyPackages]() if (linked) { + await this.#cleanOrphanedStoreEntries() // swap back in the idealTree // so that the lockfile is preserved this.idealTree = oldTree } await this[_saveIdealTree](options) + this.#linkedActualForDiff = null // clean up any trash that is still in the tree for (const path of this[_trashList]) { const loc = relpath(this.idealTree.realpath, path) @@ -161,7 +167,7 @@ module.exports = cls => class Reifier extends cls { // was not changed, delete anything in the ideal and not actual. // Then we move the entire idealTree over to this.actualTree, and // save the hidden lockfile. - if (this.diff && this.diff.filterSet.size) { + if (this.diff && this.diff.filterSet.size && !linked) { const reroot = new Set() const { filterSet } = this.diff @@ -442,9 +448,14 @@ module.exports = cls => class Reifier extends cls { if (ideal) { filterNodes.push(ideal) } - const actual = this.actualTree.children.get(ws) - if (actual) { - filterNodes.push(actual) + // Skip actual-side filterNodes when using the linked diff wrapper. + // Those nodes have root===actualTree, not root===linkedActualForDiff, and Diff.calculate requires filterNode.root to match actual. + // The ideal filterNode alone is sufficient to scope the workspace diff. + if (!this.#linkedActualForDiff) { + const actual = this.actualTree.children.get(ws) + if (actual) { + filterNodes.push(actual) + } } } } @@ -465,7 +476,7 @@ module.exports = cls => class Reifier extends cls { this.diff = Diff.calculate({ shrinkwrapInflated: this.#shrinkwrapInflated, filterNodes, - actual: this.actualTree, + actual: this.#linkedActualForDiff || this.actualTree, ideal: this.idealTree, }) @@ -625,6 +636,7 @@ module.exports = cls => class Reifier extends cls { // if the directory already exists, made will be undefined. if that's the case // we don't want to remove it because we aren't the ones who created it so we // omit it from the #sparseTreeRoots + /* istanbul ignore next -- pre-existing: mkdir returns undefined when dir exists, covered in reify tests but lost in aggregate coverage merge */ if (made) { this.#sparseTreeRoots.add(made) } @@ -824,6 +836,125 @@ module.exports = cls => class Reifier extends cls { }) : p).then(() => node) } + // Build a flat actual tree wrapper for linked installs so the diff can + // correctly match store entries that already exist on disk. + #buildLinkedActualForDiff (idealTree, actualTree) { + const combined = new Map() + + for (const child of actualTree.children.values()) { + combined.set(child.path, child) + } + + for (const child of idealTree.children.values()) { + if (!combined.has(child.path) && (child.isInStore || child.isStoreLink) && + existsSync(child.path)) { + const entry = { + global: false, + globalTop: false, + isProjectRoot: false, + isTop: false, + location: child.location, + name: child.name, + optional: child.optional, + top: child.top, + children: [], + edgesIn: new Set(), + edgesOut: new Map(), + binPaths: [], + fsChildren: [], + /* istanbul ignore next -- emulate Node */ + getBundler () { + return null + }, + hasShrinkwrap: false, + inDepBundle: false, + integrity: null, + isLink: Boolean(child.isLink), + isRoot: false, + isInStore: Boolean(child.isInStore), + path: child.path, + realpath: child.realpath, + resolved: child.resolved, + version: child.version, + package: child.package, + } + entry.target = entry + if (child.isLink && combined.has(child.realpath)) { + entry.target = combined.get(child.realpath) + } + combined.set(child.path, entry) + } + } + + const origGet = actualTree.children.get.bind(actualTree.children) + const combinedGet = combined.get.bind(combined) + /* istanbul ignore next -- only reached during scoped workspace installs */ + combined.get = (key) => combinedGet(key) || origGet(key) + + const wrapper = { + isRoot: true, + isLink: actualTree.isLink, + target: actualTree.target, + fsChildren: actualTree.fsChildren, + path: actualTree.path, + realpath: actualTree.realpath, + edgesOut: actualTree.edgesOut, + inventory: actualTree.inventory, + package: actualTree.package, + resolved: actualTree.resolved, + version: actualTree.version, + integrity: actualTree.integrity, + binPaths: actualTree.binPaths, + hasShrinkwrap: false, + inDepBundle: false, + parent: null, + children: combined, + } + + for (const child of combined.values()) { + if (!child.parent) { + child.parent = wrapper + child.root = wrapper + } + } + + return wrapper + } + + // After a linked install, scan node_modules/.store/ and remove any + // directories that are not referenced by the current ideal tree. + async #cleanOrphanedStoreEntries () { + const storeDir = resolve(this.path, 'node_modules', '.store') + let entries + try { + entries = await readdir(storeDir) + } catch { + return + } + + const validKeys = new Set() + for (const child of this.idealTree.children.values()) { + if (child.isInStore) { + const key = child.location.split(sep)[2] + validKeys.add(key) + } + } + + const orphaned = entries.filter(e => !validKeys.has(e)) + if (!orphaned.length) { + return + } + + log.silly('reify', 'cleaning orphaned store entries', orphaned) + await promiseAllRejectLate( + orphaned.map(e => + rm(resolve(storeDir, e), { recursive: true, force: true }) + .catch(/* istanbul ignore next -- rm with force rarely fails */ + er => log.warn('cleanup', `Failed to remove orphaned store entry ${e}`, er)) + ) + ) + } + #registryResolved (resolved) { // the default registry url is a magic value meaning "the currently // configured registry". diff --git a/workspaces/arborist/lib/diff.js b/workspaces/arborist/lib/diff.js index fb94407bb0166..e824c8855cc87 100644 --- a/workspaces/arborist/lib/diff.js +++ b/workspaces/arborist/lib/diff.js @@ -69,6 +69,7 @@ class Diff { tree: filterNode, visit: node => filterSet.add(node), getChildren: node => { + const orig = node node = node.target const loc = node.location const idealNode = ideal.inventory.get(loc) @@ -85,7 +86,12 @@ class Diff { } } - return ideals.concat(actuals) + const result = ideals.concat(actuals) + // Include link targets so store entries end up in filterSet + if (orig.isLink) { + result.push(node) + } + return result }, }) } diff --git a/workspaces/arborist/package.json b/workspaces/arborist/package.json index 04bb2f926b06e..a0bd9403fb6c1 100644 --- a/workspaces/arborist/package.json +++ b/workspaces/arborist/package.json @@ -33,6 +33,7 @@ "proggy": "^3.0.0", "promise-all-reject-late": "^1.0.0", "promise-call-limit": "^3.0.1", + "promise-retry": "^2.0.1", "read-package-json-fast": "^4.0.0", "semver": "^7.3.7", "ssri": "^12.0.0", diff --git a/workspaces/arborist/test/fixtures/isolated-nock.js b/workspaces/arborist/test/fixtures/isolated-nock.js index 6593c7f0d1935..4d08c1067c6e0 100644 --- a/workspaces/arborist/test/fixtures/isolated-nock.js +++ b/workspaces/arborist/test/fixtures/isolated-nock.js @@ -164,9 +164,13 @@ async function getRepo (graph) { // Generate the root of the graph on disk const root = graph.root const workspaces = graph.workspaces || [] + const hasScoped = workspaces.some(w => w.name.startsWith('@')) + const workspaceGlobs = hasScoped + ? ['packages/*', 'packages/@*/*'] + : ['packages/*'] const repo = { 'package.json': JSON.stringify({ - workspaces: workspaces.length !== 0 ? ['packages/*'] : undefined, + workspaces: workspaces.length !== 0 ? workspaceGlobs : undefined, ...root, }), packages: {}, @@ -192,7 +196,7 @@ function createDir (dir, structure) { Object.entries(structure).forEach(([key, value]) => { if (typeof value === 'object') { const newDir = path.join(dir, key) - fs.mkdirSync(newDir) + fs.mkdirSync(newDir, { recursive: true }) createDir(newDir, value) } else { fs.writeFileSync(path.join(dir, key), value) diff --git a/workspaces/arborist/test/isolated-mode.js b/workspaces/arborist/test/isolated-mode.js index dcc293b82dc77..492d467a9dd17 100644 --- a/workspaces/arborist/test/isolated-mode.js +++ b/workspaces/arborist/test/isolated-mode.js @@ -20,7 +20,7 @@ const { getRepo } = require('./fixtures/isolated-nock') /** * The testing framework here is work in progress, in particular it does not have nice ergonomics. - * The syntactic suggar for this framework will be introduced over time as we add more features. + * The syntactic sugar for this framework will be introduced over time as we add more features. * * The framework has two parts: * - Mocking: The tool generates a test repo based on a declarative list of packages. @@ -309,6 +309,108 @@ tap.test('simple peer dependencies scenarios', async t => { rule7.apply(t, dir, resolved, asserted) }) +tap.test('peer dependencies with legacyPeerDeps', async t => { + /* + * With legacyPeerDeps, peer dep edges are not created on the node. + * The linked strategy should still place peer deps alongside the + * package in the store so require() works from the real path. + * + * root -> phpegjs + * phpegjs -> pegjs (peer dep, no edge with legacyPeerDeps) + * root -> pegjs + */ + + const graph = { + registry: [ + { name: 'phpegjs', version: '1.0.0', peerDependencies: { pegjs: '*', missing: '*' } }, + { name: 'pegjs', version: '2.0.0' }, + { + name: 'adapter', + version: '1.0.0', + dependencies: { pegjs: '*' }, + peerDependencies: { pegjs: '*' }, + }, + ], + root: { + name: 'foo', + version: '1.2.3', + dependencies: { phpegjs: '1.0.0', pegjs: '2.0.0', adapter: '1.0.0' }, + }, + } + + const resolved = { + 'foo@1.2.3 (root)': { + 'phpegjs@1.0.0': { + 'pegjs@2.0.0 (peer)': {}, + }, + 'pegjs@2.0.0': {}, + 'adapter@1.0.0': { + 'pegjs@2.0.0': {}, + }, + }, + } + + const { dir, registry } = await getRepo(graph) + + const cache = fs.mkdtempSync(`${getTempDir()}/test-`) + const arborist = new Arborist({ path: dir, registry, packumentCache: new Map(), cache, legacyPeerDeps: true }) + await arborist.reify({ installStrategy: 'linked' }) + + // phpegjs should be able to require its peer dep pegjs + t.ok(setupRequire(dir)('phpegjs', 'pegjs'), + 'phpegjs can require peer dep pegjs with legacyPeerDeps') + + const asserted = new Set() + rule1.apply(t, dir, resolved, asserted) + rule2.apply(t, dir, resolved, asserted) + rule3.apply(t, dir, resolved, asserted) + rule5.apply(t, dir, resolved, asserted) + rule7.apply(t, dir, resolved, asserted) +}) + +tap.test('idempotent install with legacyPeerDeps and workspace peer deps', async t => { + // Regression: when legacyPeerDeps is enabled and a workspace has a peer dependency on another workspace, node.resolve() returns the Link node (not its target). This caused workspaceProxy to be called with the Link, producing store links under node_modules//node_modules/ that race with the workspace symlink at node_modules/, hitting EEXIST on the second install. + // Use many workspaces with cross-peer-deps to increase concurrency and make the race window large enough to trigger reliably. + const workspaces = [] + for (let i = 0; i < 20; i++) { + workspaces.push({ + name: `ws-${i}`, + version: '1.0.0', + dependencies: { abbrev: '1.0.0' }, + peerDependencies: { [`ws-${(i + 1) % 20}`]: '*' }, + }) + } + + const graph = { + registry: [ + { name: 'abbrev', version: '1.0.0' }, + ], + root: { + name: 'myroot', version: '1.0.0', + }, + workspaces, + } + + const { dir, registry } = await getRepo(graph) + + const cache = fs.mkdtempSync(`${getTempDir()}/test-`) + const opts = { path: dir, registry, packumentCache: new Map(), cache, legacyPeerDeps: true } + + // First install + const arb1 = new Arborist(opts) + await arb1.reify({ installStrategy: 'linked' }) + + // Second install must not throw EEXIST + const arb2 = new Arborist({ ...opts, packumentCache: new Map() }) + await arb2.reify({ installStrategy: 'linked' }) + + // Workspace symlinks should still be symlinks (not directories) + for (let i = 0; i < 20; i++) { + t.ok(fs.lstatSync(path.join(dir, 'node_modules', `ws-${i}`)).isSymbolicLink(), + `ws-${i} is still a symlink after second install`) + } +}) + tap.test('Lock file is same in hoisted and in isolated mode', async t => { const graph = { registry: [ @@ -334,7 +436,7 @@ tap.test('Lock file is same in hoisted and in isolated mode', async t => { fs.promises.readFile(path.join(isolatedModeDir, 'package-lock.json'), { encoding: 'utf8' }), ]) - t.same(hoistedModeLockFile, isolatedModeLockFile, 'hoited mode and isolated mode produce the same lockfile') + t.same(hoistedModeLockFile, isolatedModeLockFile, 'hoisted mode and isolated mode produce the same lockfile') }) tap.test('Basic workspaces setup', async t => { @@ -804,7 +906,7 @@ tap.test('shrinkwrap with peer dependencies', async t => { const arborist = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) await arborist.reify({ installStrategy: 'linked' }) - // TODO: greate the resolved object + // TODO: create the resolved object const asserted = new Set() rule1.apply(t, dir, resolved, asserted) rule2.apply(t, dir, resolved, asserted) @@ -1020,7 +1122,7 @@ tap.test('nested bundled dependencies of workspaces with conflicting isolated de } // the isexe that is bundled is hoisted - // the 'which' that is bundled is not hoisted due to a conflaict + // the 'which' that is bundled is not hoisted due to a conflict const resolved = { 'dog@1.2.3 (root)': { 'bar@1.0.0 (workspace)': { @@ -1300,6 +1402,113 @@ tap.test('scoped package', async t => { rule7.apply(t, dir, resolved, asserted) }) +tap.test('scoped workspace packages', async t => { + /* + * Dependency graph: + * + * root -> @scope/package-b (workspace) + * @scope/package-b -> @scope/package-a (workspace) + * root -> @scope/package-a (workspace) + */ + + const graph = { + registry: [ + { name: 'which', version: '1.0.0' }, + ], + root: { + name: 'myproject', version: '1.0.0', dependencies: { '@scope/package-a': '*', '@scope/package-b': '*' }, + }, + workspaces: [ + { name: '@scope/package-a', version: '1.0.0', dependencies: { which: '1.0.0' } }, + { name: '@scope/package-b', version: '1.0.0', dependencies: { '@scope/package-a': '*' } }, + ], + } + + const resolved = { + 'myproject@1.0.0 (root)': { + '@scope/package-a@1.0.0 (workspace)': { + 'which@1.0.0': {}, + }, + '@scope/package-b@1.0.0 (workspace)': { + '@scope/package-a@1.0.0 (workspace)': { + 'which@1.0.0': {}, + }, + }, + }, + } + + const { dir, registry } = await getRepo(graph) + + const cache = fs.mkdtempSync(`${getTempDir()}/test-`) + const arborist = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + await arborist.reify({ installStrategy: 'linked' }) + + const asserted = new Set() + rule1.apply(t, dir, resolved, asserted) + rule2.apply(t, dir, resolved, asserted) + rule3.apply(t, dir, resolved, asserted) + rule4.apply(t, dir, resolved, asserted) + rule7.apply(t, dir, resolved, asserted) +}) + +tap.test('aliased packages in workspace', async t => { + /* + * Dependency graph: + * + * root -> prettier (alias for npm:custom-prettier@3.0.3) + * custom-prettier -> isexe + * root -> my-pkg (workspace) + * my-pkg -> prettier (alias for npm:custom-prettier@3.0.3) + */ + + const graph = { + registry: [ + { name: 'custom-prettier', version: '3.0.3', dependencies: { isexe: '1.0.0' } }, + { name: 'isexe', version: '1.0.0' }, + ], + root: { + name: 'myproject', + version: '1.0.0', + dependencies: { prettier: 'npm:custom-prettier@3.0.3' }, + }, + workspaces: [ + { name: 'my-pkg', version: '1.0.0', dependencies: { prettier: 'npm:custom-prettier@3.0.3' } }, + ], + } + + const resolved = { + 'myproject@1.0.0 (root)': { + 'prettier@3.0.3': { + 'isexe@1.0.0': {}, + }, + 'my-pkg@1.0.0 (workspace)': { + 'prettier@3.0.3': { + 'isexe@1.0.0': {}, + }, + }, + }, + } + + const { dir, registry } = await getRepo(graph) + + const cache = fs.mkdtempSync(`${getTempDir()}/test-`) + const arborist = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + await arborist.reify({ installStrategy: 'linked' }) + + // Verify symlink uses alias name, not real package name + t.ok(setupRequire(dir)('prettier'), 'root can require via alias "prettier"') + t.notOk( + pathExists(path.join(dir, 'node_modules', 'custom-prettier')), + 'no custom-prettier symlink at root node_modules' + ) + + const asserted = new Set() + rule1.apply(t, dir, resolved, asserted) + rule2.apply(t, dir, resolved, asserted) + rule3.apply(t, dir, resolved, asserted) + rule7.apply(t, dir, resolved, asserted) +}) + tap.test('failing optional peer deps are not installed', async t => { // Input of arborist const graph = { @@ -1405,6 +1614,92 @@ tap.test('postinstall scripts are run', async t => { t.ok(postInstallRanBar) }) +tap.test('postinstall scripts run once for store packages', async t => { + // Regression test: store links should not cause scripts to run twice. + // The store entry and its symlink both end up in the build queue, but + // only the store entry should run scripts. + const graph = { + registry: [ + { + name: 'which', + version: '1.0.0', + scripts: { + postinstall: 'node -e "var c=0;try{c=+fs.readFileSync(\'postinstall-count\',\'utf8\')}catch(e){};fs.writeFileSync(\'postinstall-count\',String(c+1))"', + }, + }, + ], + root: { + name: 'foo', version: '1.2.3', dependencies: { which: '1.0.0' }, + }, + } + + const { dir, registry } = await getRepo(graph) + + const cache = fs.mkdtempSync(`${getTempDir()}/test-`) + const arborist = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + await arborist.reify({ installStrategy: 'linked' }) + + const whichDir = setupRequire(dir)('which') + const count = Number(fs.readFileSync(`${whichDir}/postinstall-count`, 'utf8')) + t.equal(count, 1, 'postinstall ran exactly once') +}) + +tap.test('workspace-filtered install with linked strategy', async t => { + // Two workspaces sharing the same dependency must not crash when installing with --workspace + --install-strategy=linked. + const graph = { + registry: [ + { name: 'abbrev', version: '2.0.0' }, + ], + root: { + name: 'myroot', version: '1.0.0', + }, + workspaces: [ + { name: 'ws-a', version: '1.0.0', dependencies: { abbrev: '2.0.0' } }, + { name: 'ws-b', version: '1.0.0', dependencies: { abbrev: '2.0.0' } }, + ], + } + + const { dir, registry } = await getRepo(graph) + + const cache = fs.mkdtempSync(`${getTempDir()}/test-`) + const arborist = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + + // Full install first + await arborist.reify({ installStrategy: 'linked' }) + + // Verify store entry exists + const storeDir = path.join(dir, 'node_modules', '.store') + const storeEntries = fs.readdirSync(storeDir) + t.ok(storeEntries.some(e => e.startsWith('abbrev@')), 'store has abbrev entry after full install') + + // Workspace-filtered install must not crash + const arborist2 = new Arborist({ + path: dir, + registry, + packumentCache: new Map(), + cache, + workspaces: ['ws-a'], + }) + await arborist2.reify({ + installStrategy: 'linked', + workspaces: ['ws-a'], + }) + + // Verify workspace filtering was actually applied (not silently skipped) + t.ok(arborist2.diff.filterSet.size > 0, 'workspace filter was applied to diff') + + // Store entries still intact + const storeEntries2 = fs.readdirSync(storeDir) + t.ok(storeEntries2.some(e => e.startsWith('abbrev@')), 'store entries preserved after ws install') + + // Workspace symlinks preserved + const wsALink = fs.readlinkSync(path.join(dir, 'packages', 'ws-a', 'node_modules', 'abbrev')) + t.ok(wsALink.includes('.store'), 'workspace a abbrev symlink points to store') + + const wsBLink = fs.readlinkSync(path.join(dir, 'packages', 'ws-b', 'node_modules', 'abbrev')) + t.ok(wsBLink.includes('.store'), 'workspace b abbrev symlink preserved') +}) + tap.test('bins are installed', async t => { // Input of arborist const graph = { @@ -1441,6 +1736,202 @@ tap.test('bins are installed', async t => { t.ok(binFromBarToWhich) }) +tap.test('file: dependency with linked strategy', async t => { + /* + * Regression test for https://github.com/npm/cli/issues/7549 + * + * A relative file: dependency (file:./project2) was incorrectly resolved as file:../project2, causing ENOENT errors because the path was resolved one level above the project root. + */ + const graph = { + registry: [], + root: { + name: 'project1', + version: '1.0.0', + dependencies: { project2: 'file:./project2' }, + }, + } + + const { dir, registry } = await getRepo(graph) + + // Create the local file: dependency on disk + const depDir = path.join(dir, 'project2') + fs.mkdirSync(depDir, { recursive: true }) + fs.writeFileSync(path.join(depDir, 'package.json'), JSON.stringify({ + name: 'project2', + version: '1.0.0', + })) + fs.writeFileSync(path.join(depDir, 'index.js'), "module.exports = 'project2'") + + const cache = fs.mkdtempSync(`${getTempDir()}/test-`) + const arborist = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + await arborist.reify({ installStrategy: 'linked' }) + + // The file dep should be symlinked in node_modules + const linkPath = path.join(dir, 'node_modules', 'project2') + const stat = fs.lstatSync(linkPath) + t.ok(stat.isSymbolicLink(), 'project2 is a symlink in node_modules') + + // The symlink should resolve to the actual local directory + const realpath = fs.realpathSync(linkPath) + t.equal(realpath, depDir, 'symlink points to the correct local directory') + + // The package should be requireable + t.ok(setupRequire(dir)('project2'), 'project2 can be required from root') +}) + +tap.test('subsequent linked install is a no-op', async t => { + const graph = { + registry: [ + { name: 'which', version: '1.0.0', bin: './bin.js', dependencies: { isexe: '^1.0.0' } }, + { name: 'isexe', version: '1.0.0' }, + ], + root: { + name: 'myproject', + version: '1.0.0', + dependencies: { which: '1.0.0' }, + }, + } + const { dir, registry } = await getRepo(graph) + const cache = fs.mkdtempSync(`${getTempDir()}/test-`) + + // First install + const arb1 = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + await arb1.reify({ installStrategy: 'linked' }) + + // Verify packages are installed + t.ok(fs.lstatSync(path.join(dir, 'node_modules', 'which')).isSymbolicLink(), + 'which is a symlink after first install') + + // Second install — should detect everything is up-to-date + const arb2 = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + await arb2.reify({ installStrategy: 'linked' }) + + // Verify the diff has zero actionable leaves + const leaves = arb2.diff?.leaves || [] + const actions = leaves.filter(l => l.action) + t.equal(actions.length, 0, 'second install should have no diff actions') + + // Verify unchanged nodes were detected + t.ok(arb2.diff.unchanged.length > 0, 'second install should have unchanged nodes') + + // Verify packages are still correctly installed + t.ok(fs.lstatSync(path.join(dir, 'node_modules', 'which')).isSymbolicLink(), + 'which is still a symlink after second install') + t.ok(setupRequire(dir)('which'), 'which is requireable after second install') +}) + +tap.test('workspace links are not affected by store resolved fix', async t => { + const graph = { + registry: [ + { name: 'abbrev', version: '1.0.0' }, + ], + root: { + name: 'myproject', + version: '1.0.0', + dependencies: { abbrev: '1.0.0' }, + }, + workspaces: [ + { name: 'mypkg', version: '1.0.0', dependencies: { abbrev: '1.0.0' } }, + ], + } + const { dir, registry } = await getRepo(graph) + const cache = fs.mkdtempSync(`${getTempDir()}/test-`) + + // First install + const arb1 = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + await arb1.reify({ installStrategy: 'linked' }) + + // Second install + const arb2 = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + await arb2.reify({ installStrategy: 'linked' }) + + // Verify workspace is still correctly linked + t.ok(setupRequire(dir)('mypkg'), 'workspace is requireable after second install') + t.ok(setupRequire(dir)('abbrev'), 'registry dep is requireable after second install') + + // Verify the diff has unchanged nodes (store entries are correctly matched) + t.ok(arb2.diff.unchanged.length > 0, 'second install should have unchanged nodes') +}) + +tap.test('orphaned store entries are cleaned up on dependency update', async t => { + const graph = { + registry: [ + { name: 'which', version: '1.0.0', dependencies: { isexe: '^1.0.0' } }, + { name: 'which', version: '2.0.0', dependencies: { isexe: '^1.0.0' } }, + { name: 'isexe', version: '1.0.0' }, + ], + root: { + name: 'myproject', + version: '1.0.0', + dependencies: { which: '1.0.0' }, + }, + } + const { dir, registry } = await getRepo(graph) + const cache = fs.mkdtempSync(`${getTempDir()}/test-`) + const storeDir = path.join(dir, 'node_modules', '.store') + + // First install — which@1.0.0 + const arb1 = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + await arb1.reify({ installStrategy: 'linked' }) + + const entriesAfterV1 = fs.readdirSync(storeDir) + t.ok(entriesAfterV1.some(e => e.startsWith('which@1.0.0-')), + 'store has which@1.0.0 entry after first install') + + // Update package.json to depend on which@2.0.0 + const pkgPath = path.join(dir, 'package.json') + const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8')) + pkg.dependencies.which = '2.0.0' + fs.writeFileSync(pkgPath, JSON.stringify(pkg)) + + // Second install — which@2.0.0 + const arb2 = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + await arb2.reify({ installStrategy: 'linked' }) + + const entriesAfterV2 = fs.readdirSync(storeDir) + t.ok(entriesAfterV2.some(e => e.startsWith('which@2.0.0-')), + 'store has which@2.0.0 entry after update') + t.notOk(entriesAfterV2.some(e => e.startsWith('which@1.0.0-')), + 'old which@1.0.0 store entry is removed after update') +}) + +tap.test('orphaned store entries are cleaned up on dependency removal', async t => { + const graph = { + registry: [ + { name: 'which', version: '1.0.0', dependencies: { isexe: '^1.0.0' } }, + { name: 'isexe', version: '1.0.0' }, + ], + root: { + name: 'myproject', + version: '1.0.0', + dependencies: { which: '1.0.0' }, + }, + } + const { dir, registry } = await getRepo(graph) + const cache = fs.mkdtempSync(`${getTempDir()}/test-`) + const storeDir = path.join(dir, 'node_modules', '.store') + + // First install + const arb1 = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + await arb1.reify({ installStrategy: 'linked' }) + + t.ok(fs.readdirSync(storeDir).length > 0, 'store has entries after install') + + // Remove the dependency + const pkgPath = path.join(dir, 'package.json') + const pkg = JSON.parse(fs.readFileSync(pkgPath, 'utf8')) + delete pkg.dependencies + fs.writeFileSync(pkgPath, JSON.stringify(pkg)) + + // Reinstall + const arb2 = new Arborist({ path: dir, registry, packumentCache: new Map(), cache }) + await arb2.reify({ installStrategy: 'linked' }) + + const entriesAfterRemoval = fs.readdirSync(storeDir) + t.equal(entriesAfterRemoval.length, 0, + 'all store entries are removed when dependencies are removed') +}) + function setupRequire (cwd) { return function requireChain (...chain) { return chain.reduce((path, name) => {