codekingpro/portable-devtools
114k
1// mixin implementing the reify method
2const PackageJson = require('@npmcli/package-json')
3const hgi = require('hosted-git-info')
4const npa = require('npm-package-arg')
5const packageContents = require('@npmcli/installed-package-contents')
6const pacote = require('pacote')
7const promiseAllRejectLate = require('promise-all-reject-late')
8const runScript = require('@npmcli/run-script')
9const { callLimit: promiseCallLimit } = require('promise-call-limit')
10const { depth: dfwalk } = require('treeverse')
11const { dirname, resolve, relative, join, sep } = require('node:path')
12const { log, time } = require('proc-log')
13const { existsSync } = require('node:fs')
14const { lstat, mkdir, readdir, rm, symlink } = require('node:fs/promises')
15const { moveFile } = require('@npmcli/fs')
16const { subset, intersects } = require('semver')
17const { walkUp } = require('walk-up-path')
18
19const AuditReport = require('../audit-report.js')
20const Diff = require('../diff.js')
21const calcDepFlags = require('../calc-dep-flags.js')
22const debug = require('../debug.js')
23const onExit = require('../signal-handling.js')
24const optionalSet = require('../optional-set.js')
25const relpath = require('../relpath.js')
26const retirePath = require('../retire-path.js')
27const treeCheck = require('../tree-check.js')
28const { defaultLockfileVersion } = require('../shrinkwrap.js')
29const { saveTypeMap, hasSubKey } = require('../add-rm-pkg-deps.js')
30const { IsolatedNode, IsolatedLink } = require('../isolated-classes.js')
31
32// Part of steps (steps need refactoring before we can do anything about these)
33const _retireShallowNodes = Symbol.for('retireShallowNodes')
34const _loadBundlesAndUpdateTrees = Symbol.for('loadBundlesAndUpdateTrees')
35const _submitQuickAudit = Symbol('submitQuickAudit')
36const _unpackNewModules = Symbol.for('unpackNewModules')
37const _build = Symbol.for('build')
38
39// shared by rebuild mixin
40const _trashList = Symbol.for('trashList')
41const _handleOptionalFailure = Symbol.for('handleOptionalFailure')
42const _loadTrees = Symbol.for('loadTrees')
43// defined by rebuild mixin
44const _checkBins = Symbol.for('checkBins')
45
46// shared symbols for swapping out when testing
47// TODO tests should not be this deep into internals
48const _diffTrees = Symbol.for('diffTrees')
49const _createSparseTree = Symbol.for('createSparseTree')
50const _loadShrinkwrapsAndUpdateTrees = Symbol.for('loadShrinkwrapsAndUpdateTrees')
51const _reifyNode = Symbol.for('reifyNode')
52const _updateAll = Symbol.for('updateAll')
53const _updateNames = Symbol.for('updateNames')
54const _moveContents = Symbol.for('moveContents')
55const _moveBackRetiredUnchanged = Symbol.for('moveBackRetiredUnchanged')
56const _removeTrash = Symbol.for('removeTrash')
57const _renamePath = Symbol.for('renamePath')
58const _rollbackRetireShallowNodes = Symbol.for('rollbackRetireShallowNodes')
59const _rollbackCreateSparseTree = Symbol.for('rollbackCreateSparseTree')
60const _rollbackMoveBackRetiredUnchanged = Symbol.for('rollbackMoveBackRetiredUnchanged')
61const _saveIdealTree = Symbol.for('saveIdealTree')
62
63// defined by build-ideal-tree mixin
64const _resolvedAdd = Symbol.for('resolvedAdd')
65// used by build-ideal-tree mixin
66const _addNodeToTrashList = Symbol.for('addNodeToTrashList')
67
68module.exports = cls => class Reifier extends cls {
69 #bundleMissing = new Set() // child nodes we'd EXPECT to be included in a bundle, but aren't
70 #bundleUnpacked = new Set() // the nodes we unpack to read their bundles
71 #nmValidated = new Set()
72 #omit
73 #retiredPaths = {}
74 #retiredUnchanged = {}
75 #shrinkwrapInflated = new Set()
76 #sparseTreeDirs = new Set()
77 #sparseTreeRoots = new Set()
78 #linkedActualForDiff = null
79
80 constructor (options) {
81 super(options)
82
83 this[_trashList] = new Set()
84 }
85
86 // public method
87 async reify (options = {}) {
88 const linked = (options.installStrategy || this.options.installStrategy) === 'linked'
89
90 if (this.options.packageLockOnly && this.options.global) {
91 const er = new Error('cannot generate lockfile for global packages')
92 er.code = 'ESHRINKWRAPGLOBAL'
93 throw er
94 }
95
96 this.#omit = new Set(options.omit)
97
98 // start tracker block
99 this.addTracker('reify')
100 const timeEnd = time.start('reify')
101 // don't create missing dirs on dry runs
102 if (!this.options.packageLockOnly && !this.options.dryRun) {
103 // we do NOT want to set ownership on this folder, especially
104 // recursively, because it can have other side effects to do that
105 // in a project directory. We just want to make it if it's missing.
106 await mkdir(resolve(this.path), { recursive: true })
107
108 // do not allow the top-level node_modules to be a symlink
109 await this.#validateNodeModules(resolve(this.path, 'node_modules'))
110 }
111 await this[_loadTrees](options)
112
113 const oldTree = this.idealTree
114 if (linked) {
115 // swap out the tree with the isolated tree
116 // this is currently technical debt which will be resolved in a refactor
117 // of Node/Link trees
118 log.warn('reify', 'The "linked" install strategy is EXPERIMENTAL and may contain bugs.')
119 this.idealTree = await this.createIsolatedTree()
120 if (this.actualTree) {
121 this.#linkedActualForDiff = this.#buildLinkedActualForDiff(
122 this.idealTree, this.actualTree
123 )
124 }
125 }
126 await this[_diffTrees]()
127 await this.#reifyPackages()
128 if (linked) {
129 await this.#cleanOrphanedStoreEntries()
130 // swap back in the idealTree
131 // so that the lockfile is preserved
132 this.idealTree = oldTree
133 }
134 await this[_saveIdealTree](options)
135 this.#linkedActualForDiff = null
136 // clean inert
137 for (const node of this.idealTree.inventory.values()) {
138 if (node.inert) {
139 node.parent = null
140 }
141 }
142 // clean up any trash that is still in the tree
143 for (const path of this[_trashList]) {
144 const loc = relpath(this.idealTree.realpath, path)
145 const node = this.idealTree.inventory.get(loc)
146 if (node && node.root === this.idealTree) {
147 node.parent = null
148 }
149 }
150
151 // if we filtered to only certain nodes, then anything ELSE needs
152 // to be untouched in the resulting actual tree, even if it differs
153 // in the idealTree. Copy over anything that was in the actual and
154 // was not changed, delete anything in the ideal and not actual.
155 // Then we move the entire idealTree over to this.actualTree, and
156 // save the hidden lockfile.
157 if (this.diff && this.diff.filterSet.size && !linked) {
158 const reroot = new Set()
159
160 const { filterSet } = this.diff
161 const seen = new Set()
162 for (const [loc, ideal] of this.idealTree.inventory.entries()) {
163 seen.add(loc)
164
165 // if it's an ideal node from the filter set, then skip it
166 // because we already made whatever changes were necessary
167 if (filterSet.has(ideal)) {
168 continue
169 }
170
171 // otherwise, if it's not in the actualTree, then it's not a thing
172 // that we actually added. And if it IS in the actualTree, then
173 // it's something that we left untouched, so we need to record
174 // that.
175 const actual = this.actualTree.inventory.get(loc)
176 if (!actual) {
177 ideal.root = null
178 } else {
179 if ([...actual.linksIn].some(link => filterSet.has(link))) {
180 seen.add(actual.location)
181 continue
182 }
183 const { realpath, isLink } = actual
184 if (isLink && ideal.isLink && ideal.realpath === realpath) {
185 continue
186 } else {
187 reroot.add(actual)
188 }
189 }
190 }
191
192 // now find any actual nodes that may not be present in the ideal
193 // tree, but were left behind by virtue of not being in the filter
194 for (const [loc, actual] of this.actualTree.inventory.entries()) {
195 if (seen.has(loc)) {
196 continue
197 }
198 seen.add(loc)
199
200 // we know that this is something that ISN'T in the idealTree,
201 // or else we will have addressed it in the previous loop.
202 // If it's in the filterSet, that means we intentionally removed
203 // it, so nothing to do here.
204 if (filterSet.has(actual)) {
205 continue
206 }
207
208 reroot.add(actual)
209 }
210
211 // go through the rerooted actual nodes, and move them over.
212 for (const actual of reroot) {
213 actual.root = this.idealTree
214 }
215
216 // prune out any tops that lack a linkIn, they are no longer relevant.
217 for (const top of this.idealTree.tops) {
218 if (top.linksIn.size === 0) {
219 top.root = null
220 }
221 }
222
223 // need to calculate dep flags, since nodes may have been marked
224 // as extraneous or otherwise incorrect during transit.
225 calcDepFlags(this.idealTree)
226 }
227
228 // save the ideal's meta as a hidden lockfile after we actualize it
229 this.idealTree.meta.filename =
230 this.idealTree.realpath + '/node_modules/.package-lock.json'
231 this.idealTree.meta.hiddenLockfile = true
232 this.idealTree.meta.lockfileVersion = defaultLockfileVersion
233
234 this.actualTree = this.idealTree
235 this.idealTree = null
236
237 if (!this.options.global) {
238 await this.actualTree.meta.save()
239 const ignoreScripts = !!this.options.ignoreScripts
240 // if we aren't doing a dry run or ignoring scripts and we actually made changes to the dep
241 // tree, then run the dependencies scripts
242 if (!this.options.dryRun && !ignoreScripts && this.diff && this.diff.children.length) {
243 const { path, package: pkg } = this.actualTree.target
244 const stdio = this.options.foregroundScripts ? 'inherit' : 'pipe'
245 const { scripts = {} } = pkg
246 for (const event of ['predependencies', 'dependencies', 'postdependencies']) {
247 if (Object.prototype.hasOwnProperty.call(scripts, event)) {
248 log.info('run', pkg._id, event, scripts[event])
249 await time.start(`reify:run:${event}`, () => runScript({
250 event,
251 path,
252 pkg,
253 stdio,
254 scriptShell: this.options.scriptShell,
255 }))
256 }
257 }
258 }
259 }
260 // This is a very bad pattern and I can't wait to stop doing it
261 this.auditReport = await this.auditReport
262
263 this.finishTracker('reify')
264 timeEnd()
265 return treeCheck(this.actualTree)
266 }
267
268 async #reifyPackages () {
269 // we don't submit the audit report or write to disk on dry runs
270 if (this.options.dryRun) {
271 return
272 }
273
274 if (this.options.packageLockOnly) {
275 // we already have the complete tree, so just audit it now,
276 // and that's all we have to do here.
277 return this[_submitQuickAudit]()
278 }
279
280 // ok, we're about to start touching the fs. need to roll back
281 // if we get an early termination.
282 let reifyTerminated = null
283 const removeHandler = onExit(({ signal }) => {
284 // only call once. if signal hits twice, we just terminate
285 removeHandler()
286 reifyTerminated = Object.assign(new Error('process terminated'), {
287 signal,
288 })
289 return false
290 })
291
292 // [rollbackfn, [...actions]]
293 // after each step, if the process was terminated, execute the rollback
294 // note that each rollback *also* calls the previous one when it's
295 // finished, and then the first one throws the error, so we only need
296 // a new rollback step when we have a new thing that must be done to
297 // revert the install.
298 const steps = [
299 [_rollbackRetireShallowNodes, [
300 _retireShallowNodes,
301 ]],
302 [_rollbackCreateSparseTree, [
303 _createSparseTree,
304 _loadShrinkwrapsAndUpdateTrees,
305 _loadBundlesAndUpdateTrees,
306 _submitQuickAudit,
307 _unpackNewModules,
308 ]],
309 [_rollbackMoveBackRetiredUnchanged, [
310 _moveBackRetiredUnchanged,
311 _build,
312 ]],
313 ]
314 for (const [rollback, actions] of steps) {
315 for (const action of actions) {
316 try {
317 await this[action]()
318 if (reifyTerminated) {
319 throw reifyTerminated
320 }
321 } catch (er) {
322 // TODO rollbacks shouldn't be relied on to throw err
323 await this[rollback](er)
324 /* istanbul ignore next - rollback throws, should never hit this */
325 throw er
326 }
327 }
328 }
329
330 // no rollback for this one, just exit with the error, since the
331 // install completed and can't be safely recovered at this point.
332 await this[_removeTrash]()
333 if (reifyTerminated) {
334 throw reifyTerminated
335 }
336
337 // done modifying the file system, no need to keep listening for sigs
338 removeHandler()
339 }
340
341 // when doing a local install, we load everything and figure it all out.
342 // when doing a global install, we *only* care about the explicit requests.
343 [_loadTrees] (options) {
344 const timeEnd = time.start('reify:loadTrees')
345 const bitOpt = {
346 ...options,
347 complete: this.options.packageLockOnly || this.options.dryRun,
348 }
349
350 // if we're only writing a package lock, then it doesn't matter what's here
351 if (this.options.packageLockOnly) {
352 return this.buildIdealTree(bitOpt).then(timeEnd)
353 }
354
355 const actualOpt = this.options.global ? {
356 ignoreMissing: true,
357 global: true,
358 filter: (node, kid) => {
359 // if it's not the project root, and we have no explicit requests,
360 // then we're already into a nested dep, so we keep it
361 if (this.explicitRequests.size === 0 || !node.isProjectRoot) {
362 return true
363 }
364
365 // if we added it as an edgeOut, then we want it
366 if (this.idealTree.edgesOut.has(kid)) {
367 return true
368 }
369
370 // if it's an explicit request, then we want it
371 const hasExplicit = [...this.explicitRequests]
372 .some(edge => edge.name === kid)
373 if (hasExplicit) {
374 return true
375 }
376
377 // ignore the rest of the global install folder
378 return false
379 },
380 } : { ignoreMissing: true }
381
382 if (!this.options.global) {
383 return Promise.all([
384 this.loadActual(actualOpt),
385 this.buildIdealTree(bitOpt),
386 ]).then(timeEnd)
387 }
388
389 // the global install space tends to have a lot of stuff in it. don't
390 // load all of it, just what we care about. we won't be saving a
391 // hidden lockfile in there anyway. Note that we have to load ideal
392 // BEFORE loading actual, so that the actualOpt can use the
393 // explicitRequests which is set during buildIdealTree
394 return this.buildIdealTree(bitOpt)
395 .then(() => this.loadActual(actualOpt))
396 .then(timeEnd)
397 }
398
399 [_diffTrees] () {
400 if (this.options.packageLockOnly) {
401 return
402 }
403
404 const timeEnd = time.start('reify:diffTrees')
405 // XXX if we have an existing diff already, there should be a way
406 // to just invalidate the parts that changed, but avoid walking the
407 // whole tree again.
408
409 const includeWorkspaces = this.options.workspacesEnabled
410 const includeRootDeps = !includeWorkspaces
411 || this.options.includeWorkspaceRoot && this.options.workspaces.length > 0
412
413 const filterNodes = []
414 if (this.options.global && this.explicitRequests.size) {
415 const idealTree = this.idealTree.target
416 const actualTree = this.actualTree.target
417 // we ONLY are allowed to make changes in the global top-level
418 // children where there's an explicit request.
419 for (const { name } of this.explicitRequests) {
420 const ideal = idealTree.children.get(name)
421 if (ideal) {
422 filterNodes.push(ideal)
423 }
424 const actual = actualTree.children.get(name)
425 if (actual) {
426 filterNodes.push(actual)
427 }
428 }
429 } else {
430 if (includeWorkspaces) {
431 // add all ws nodes to filterNodes
432 for (const ws of this.options.workspaces) {
433 const ideal = this.idealTree.children.get(ws)
434 if (ideal) {
435 filterNodes.push(ideal)
436 }
437 // Skip actual-side filterNodes when using the linked diff wrapper.
438 // Those nodes have root===actualTree, not root===linkedActualForDiff, and Diff.calculate requires filterNode.root to match actual.
439 // The ideal filterNode alone is sufficient to scope the workspace diff.
440 if (!this.#linkedActualForDiff) {
441 const actual = this.actualTree.children.get(ws)
442 if (actual) {
443 filterNodes.push(actual)
444 }
445 }
446 }
447 }
448 if (includeRootDeps) {
449 // add all non-workspace nodes to filterNodes
450 for (const tree of [this.idealTree, this.actualTree]) {
451 for (const { type, to } of tree.edgesOut.values()) {
452 if (type !== 'workspace' && to) {
453 filterNodes.push(to)
454 }
455 }
456 }
457 }
458 }
459
460 // find all the nodes that need to change between the actual
461 // and ideal trees.
462 this.diff = Diff.calculate({
463 omit: this.#omit,
464 shrinkwrapInflated: this.#shrinkwrapInflated,
465 filterNodes,
466 actual: this.#linkedActualForDiff || this.actualTree,
467 ideal: this.idealTree,
468 })
469
470 // we don't have to add 'removed' folders to the trashlist, because
471 // they'll be moved aside to a retirement folder, and then the retired
472 // folder will be deleted at the end. This is important when we have
473 // a folder like FOO being "removed" in favor of a folder like "foo",
474 // because if we remove node_modules/FOO on case-insensitive systems,
475 // it will remove the dep that we *want* at node_modules/foo.
476
477 timeEnd()
478 }
479
480 // add the node and all its bins to the list of things to be
481 // removed later on in the process. optionally, also mark them
482 // as a retired paths, so that we move them out of the way and
483 // replace them when rolling back on failure.
484 [_addNodeToTrashList] (node, retire = false) {
485 const paths = [node.path, ...node.binPaths]
486 const moves = this.#retiredPaths
487 log.silly('reify', 'mark', retire ? 'retired' : 'deleted', paths)
488 for (const path of paths) {
489 if (retire) {
490 const retired = retirePath(path)
491 moves[path] = retired
492 this[_trashList].add(retired)
493 } else {
494 this[_trashList].add(path)
495 }
496 }
497 }
498
499 // move aside the shallowest nodes in the tree that have to be
500 // changed or removed, so that we can rollback if necessary.
501 [_retireShallowNodes] () {
502 const timeEnd = time.start('reify:retireShallow')
503 const moves = this.#retiredPaths = {}
504 for (const diff of this.diff.children) {
505 if (diff.action === 'CHANGE' || diff.action === 'REMOVE') {
506 // we'll have to clean these up at the end, so add them to the list
507 this[_addNodeToTrashList](diff.actual, true)
508 }
509 }
510 log.silly('reify', 'moves', moves)
511 const movePromises = Object.entries(moves)
512 .map(([from, to]) => this[_renamePath](from, to))
513 return promiseAllRejectLate(movePromises).then(timeEnd)
514 }
515
516 [_renamePath] (from, to, didMkdirp = false) {
517 return moveFile(from, to)
518 .catch(er => {
519 // Occasionally an expected bin file might not exist in the package,
520 // or a shim/symlink might have been moved aside. If we've already
521 // handled the most common cause of ENOENT (dir doesn't exist yet),
522 // then just ignore any ENOENT.
523 if (er.code === 'ENOENT') {
524 return didMkdirp ? null : mkdir(dirname(to), { recursive: true }).then(() =>
525 this[_renamePath](from, to, true))
526 } else if (er.code === 'EEXIST' || er.code === 'ENOTEMPTY') {
527 return rm(to, { recursive: true, force: true }).then(() => moveFile(from, to))
528 } else {
529 throw er
530 }
531 })
532 }
533
534 [_rollbackRetireShallowNodes] (er) {
535 const timeEnd = time.start('reify:rollback:retireShallow')
536 const moves = this.#retiredPaths
537 const movePromises = Object.entries(moves)
538 .map(([from, to]) => this[_renamePath](to, from))
539 return promiseAllRejectLate(movePromises)
540 // ignore subsequent rollback errors
541 .catch(() => {})
542 .then(timeEnd)
543 .then(() => {
544 throw er
545 })
546 }
547
548 [_createSparseTree] () {
549 const timeEnd = time.start('reify:createSparse')
550 // if we call this fn again, we look for the previous list
551 // so that we can avoid making the same directory multiple times
552 const leaves = this.diff.leaves
553 .filter(diff => {
554 return (diff.action === 'ADD' || diff.action === 'CHANGE') &&
555 !this.#sparseTreeDirs.has(diff.ideal.path) &&
556 !diff.ideal.isLink
557 })
558 .map(diff => diff.ideal)
559
560 // we check this in parallel, so guard against multiple attempts to
561 // retire the same path at the same time.
562 const dirsChecked = new Set()
563 return promiseAllRejectLate(leaves.map(async node => {
564 for (const d of walkUp(node.path)) {
565 if (d === node.top.path) {
566 break
567 }
568 if (dirsChecked.has(d)) {
569 continue
570 }
571 dirsChecked.add(d)
572 const st = await lstat(d).catch(() => null)
573 // this can happen if we have a link to a package with a name
574 // that the filesystem treats as if it is the same thing.
575 // would be nice to have conditional istanbul ignores here...
576 /* istanbul ignore next - defense in depth */
577 if (st && !st.isDirectory()) {
578 const retired = retirePath(d)
579 this.#retiredPaths[d] = retired
580 this[_trashList].add(retired)
581 await this[_renamePath](d, retired)
582 }
583 }
584 this.#sparseTreeDirs.add(node.path)
585 const made = await mkdir(node.path, { recursive: true })
586 // if the directory already exists, made will be undefined. if that's the case
587 // we don't want to remove it because we aren't the ones who created it so we
588 // omit it from the #sparseTreeRoots
589 /* istanbul ignore next -- pre-existing: mkdir returns undefined when dir exists, covered in reify tests but lost in aggregate coverage merge */
590 if (made) {
591 this.#sparseTreeRoots.add(made)
592 }
593 })).then(timeEnd)
594 }
595
596 [_rollbackCreateSparseTree] (er) {
597 const timeEnd = time.start('reify:rollback:createSparse')
598 // cut the roots of the sparse tree that were created, not the leaves
599 const roots = this.#sparseTreeRoots
600 // also delete the moves that we retired, so that we can move them back
601 const failures = []
602 const targets = [...roots, ...Object.keys(this.#retiredPaths)]
603 const unlinks = targets
604 .map(path => rm(path, { recursive: true, force: true }).catch(er => failures.push([path, er])))
605 return promiseAllRejectLate(unlinks).then(() => {
606 // eslint-disable-next-line promise/always-return
607 if (failures.length) {
608 log.warn('cleanup', 'Failed to remove some directories', failures)
609 }
610 })
611 .then(timeEnd)
612 .then(() => this[_rollbackRetireShallowNodes](er))
613 }
614
615 // shrinkwrap nodes define their dependency branches with a file, so
616 // we need to unpack them, read that shrinkwrap file, and then update
617 // the tree by calling loadVirtual with the node as the root.
618 [_loadShrinkwrapsAndUpdateTrees] () {
619 const seen = this.#shrinkwrapInflated
620 const shrinkwraps = this.diff.leaves
621 .filter(d => (d.action === 'CHANGE' || d.action === 'ADD' || !d.action) &&
622 d.ideal.hasShrinkwrap && !seen.has(d.ideal) &&
623 !this[_trashList].has(d.ideal.path))
624
625 if (!shrinkwraps.length) {
626 return
627 }
628
629 const timeEnd = time.start('reify:loadShrinkwraps')
630
631 const Arborist = this.constructor
632 return promiseAllRejectLate(shrinkwraps.map(diff => {
633 const node = diff.ideal
634 seen.add(node)
635 return diff.action ? this[_reifyNode](node) : node
636 }))
637 .then(nodes => promiseAllRejectLate(nodes.map(node => new Arborist({
638 ...this.options,
639 path: node.path,
640 }).loadVirtual({ root: node }))))
641 // reload the diff and sparse tree because the ideal tree changed
642 .then(() => this[_diffTrees]())
643 .then(() => this[_createSparseTree]())
644 .then(() => this[_loadShrinkwrapsAndUpdateTrees]())
645 .then(timeEnd)
646 }
647
648 // create a symlink for Links, extract for Nodes
649 // return the node object, since we usually want that
650 // handle optional dep failures here
651 // If reifying fails, and the node is optional, add it and its optionalSet
652 // to the trash list
653 // Always return the node.
654 [_reifyNode] (node) {
655 const timeEnd = time.start(`reifyNode:${node.location}`)
656 this.addTracker('reify', node.name, node.location)
657
658 const p = Promise.resolve().then(async () => {
659 await this[_checkBins](node)
660 await this.#extractOrLink(node)
661 const { _id, deprecated } = node.package
662 // The .catch is in _handleOptionalFailure. Not ideal, this should be cleaned up.
663 // eslint-disable-next-line promise/always-return
664 if (deprecated) {
665 log.warn('deprecated', `${_id}: ${deprecated}`)
666 }
667 })
668
669 return this[_handleOptionalFailure](node, p)
670 .then(() => {
671 this.finishTracker('reify', node.name, node.location)
672 timeEnd()
673 return node
674 })
675 }
676
677 // do not allow node_modules to be a symlink
678 async #validateNodeModules (nm) {
679 if (this.options.force || this.#nmValidated.has(nm)) {
680 return
681 }
682 const st = await lstat(nm).catch(() => null)
683 if (!st || st.isDirectory()) {
684 this.#nmValidated.add(nm)
685 return
686 }
687 log.warn('reify', 'Removing non-directory', nm)
688 await rm(nm, { recursive: true, force: true })
689 }
690
691 async #extractOrLink (node) {
692 const nm = resolve(node.parent.path, 'node_modules')
693 await this.#validateNodeModules(nm)
694
695 if (!node.isLink) {
696 // in normal cases, node.resolved should *always* be set by now.
697 // however, it is possible when a lockfile is damaged, or very old,
698 // or in some other race condition bugs in npm v6, that a previously
699 // bundled dependency will have just a version, but no resolved value,
700 // and no 'bundled: true' setting.
701 // Do the best with what we have, or else remove it from the tree
702 // entirely, since we can't possibly reify it.
703 let res = null
704 if (node.resolved) {
705 const registryResolved = this.#registryResolved(node.resolved)
706 if (registryResolved) {
707 res = `${node.name}@${registryResolved}`
708 }
709 } else if (node.package.name && node.version) {
710 res = `${node.package.name}@${node.version}`
711 }
712
713 // no idea what this thing is. remove it from the tree.
714 if (!res) {
715 const warning = 'invalid or damaged lockfile detected\n' +
716 'please re-try this operation once it completes\n' +
717 'so that the damage can be corrected, or perform\n' +
718 'a fresh install with no lockfile if the problem persists.'
719 log.warn('reify', warning)
720 log.verbose('reify', 'unrecognized node in tree', node.path)
721 node.parent = null
722 node.fsParent = null
723 this[_addNodeToTrashList](node)
724 return
725 }
726 await debug(async () => {
727 const st = await lstat(node.path).catch(() => null)
728 if (st && !st.isDirectory()) {
729 debug.log('unpacking into a non-directory', node)
730 throw Object.assign(new Error('ENOTDIR: not a directory'), {
731 code: 'ENOTDIR',
732 path: node.path,
733 })
734 }
735 })
736 await pacote.extract(res, node.path, {
737 ...this.options,
738 resolved: node.resolved,
739 integrity: node.integrity,
740 })
741 // store nodes don't use Node class so node.package doesn't get updated
742 if (node.isInStore) {
743 const { content: pkg } = await PackageJson.normalize(node.path)
744 node.package.scripts = pkg.scripts
745 }
746 return
747 }
748
749 // node.isLink
750 await rm(node.path, { recursive: true, force: true })
751
752 // symlink
753 const dir = dirname(node.path)
754 const target = node.realpath
755
756 let rel
757 if (node.resolved?.startsWith('file:')) {
758 rel = this.#calculateRelativePath(node, dir, target, nm)
759 } else {
760 rel = relative(dir, target)
761 }
762
763 await mkdir(dir, { recursive: true })
764 return symlink(rel, node.path, 'junction')
765 }
766
767 // if the node is optional, then the failure of the promise is nonfatal
768 // just add it and its optional set to the trash list.
769 [_handleOptionalFailure] (node, p) {
770 return (node.optional ? p.catch(() => {
771 const set = optionalSet(node)
772 for (const node of set) {
773 log.verbose('reify', 'failed optional dependency', node.path)
774 node.inert = true
775 this[_addNodeToTrashList](node)
776 }
777 }) : p).then(() => node)
778 }
779
780 #calculateRelativePath (node, dir, target) {
781 // Check if the node is affected by a root override
782 let hasRootOverride = [...node.edgesIn].some(edge => edge.from.isRoot && edge.overrides)
783 // If not set via edges, see if the root package.json explicitly lists an override
784 if (!hasRootOverride && node.root) {
785 const rootPackage = node.root.target
786 hasRootOverride = !!(rootPackage &&
787 rootPackage.package.overrides &&
788 rootPackage.package.overrides[node.name])
789 }
790 if (!hasRootOverride) {
791 return relative(dir, target)
792 }
793 // If an override is detected, attempt to retrieve the override spec from the root package.json
794 const overrideSpec = node.root?.target?.package?.overrides?.[node.name]
795 if (typeof overrideSpec === 'string' && overrideSpec.startsWith('file:')) {
796 const overridePath = overrideSpec.replace(/^file:/, '')
797 const rootDir = node.root.target.path
798 return relative(dir, resolve(rootDir, overridePath))
799 }
800
801 // Fallback: derive the file path from node.resolved in a platform-agnostic way
802 const filePath = node.resolved.replace(/^file:/, '')
803 return join(filePath)
804 }
805
806 // Build a flat actual tree wrapper for linked installs so the diff can correctly match store entries that already exist on disk.
807 // The proxy tree from createIsolatedTree() is flat (all children on root), but loadActual() produces a nested tree where store entries are deep link targets.
808 // This wrapper surfaces them at the root level for comparison.
809 #buildLinkedActualForDiff (idealTree, actualTree) {
810 // Combined Map keyed by path (how allChildren() in diff.js keys)
811 const combined = new Map()
812
813 // Create synthetic actual entries for ALL ideal children that exist on disk.
814 // The isolated ideal tree is flat (all entries as root children), but loadActual() produces a nested tree where workspace deps are under fsChildren and store entries are deep link targets.
815 // Synthetic entries ensure the diff compares matching resolved/integrity values (e.g. workspace links have resolved=undefined in the ideal tree but resolved="file:../packages/..." in the actual tree).
816 for (const child of idealTree.children.values()) {
817 if (combined.has(child.path) || !existsSync(child.path)) {
818 continue
819 }
820 // Skip store links whose ideal realpath doesn't exist on disk yet — the store hash changed and the symlink needs recreating via ADD.
821 if (child.isLink && child.resolved?.startsWith('file:.store/') && !existsSync(child.realpath)) {
822 continue
823 }
824 let entry
825 if (child.isLink) {
826 entry = new IsolatedLink(child)
827 } else {
828 entry = new IsolatedNode(child)
829 }
830 if (child.isLink && combined.has(child.realpath)) {
831 entry.target = combined.get(child.realpath)
832 }
833 combined.set(child.path, entry)
834 }
835
836 // Proxy .get(name) to original actual tree for filterNodes compatibility
837 // (scoped workspace installs use .get(name), allChildren uses .values())
838 const origGet = actualTree.children.get.bind(actualTree.children)
839 const combinedGet = combined.get.bind(combined)
840 /* istanbul ignore next -- only reached during scoped workspace installs */
841 combined.get = (key) => combinedGet(key) || origGet(key)
842
843 let wrapper
844 /* istanbul ignore next - untested! */
845 if (actualTree.isLink) {
846 wrapper = new IsolatedLink(actualTree)
847 } else {
848 wrapper = new IsolatedNode(actualTree)
849 }
850 wrapper.root = wrapper
851 wrapper.binPaths = actualTree.binPaths
852 wrapper.children = combined
853 wrapper.edgesOut = actualTree.edgesOut
854 // Use empty fsChildren so that allChildren() only picks up entries from the combined map.
855 // The actual fsChildren have real children with different resolved values (e.g. file:../../../node_modules/.store/... vs file:.store/...) that would overwrite our synthetic entries in allChildren().
856 wrapper.fsChildren = new Set()
857 wrapper.integrity = actualTree.integrity
858 wrapper.inventory = actualTree.inventory
859
860 return wrapper
861 }
862
863 #registryResolved (resolved) {
864 // the default registry url is a magic value meaning "the currently
865 // configured registry".
866 // `resolved` must never be falsey.
867 //
868 // XXX: use a magic string that isn't also a valid value, like
869 // ${REGISTRY} or something. This has to be threaded through the
870 // Shrinkwrap and Node classes carefully, so for now, just treat
871 // the default reg as the magical animal that it has been.
872 try {
873 const resolvedURL = hgi.parseUrl(resolved)
874
875 if ((this.options.replaceRegistryHost === resolvedURL.hostname) ||
876 this.options.replaceRegistryHost === 'always') {
877 const registryURL = new URL(this.registry)
878
879 // Replace the host with the registry host while keeping the path intact
880 resolvedURL.hostname = registryURL.hostname
881 resolvedURL.port = registryURL.port
882 resolvedURL.protocol = registryURL.protocol
883
884 // Make sure we don't double-include the path if it's already there
885 const registryPath = registryURL.pathname.replace(/\/$/, '')
886
887 if (registryPath && registryPath !== '/') {
888 // Check if the resolved pathname already starts with the registry path
889 // We need to ensure it's a proper path prefix, not just a string prefix
890 // e.g., registry path '/npm' should not match '/npm-run-path'
891 const hasRegistryPath = resolvedURL.pathname === registryPath ||
892 resolvedURL.pathname.startsWith(registryPath + '/')
893
894 if (!hasRegistryPath) {
895 // Since hostname is changed, we need to ensure the registry path is included
896 resolvedURL.pathname = registryPath + resolvedURL.pathname
897 }
898 }
899
900 return resolvedURL.toString()
901 }
902 return resolved
903 } catch (e) {
904 // if we could not parse the url at all then returning nothing
905 // here means it will get removed from the tree in the next step
906 return undefined
907 }
908 }
909
910 // bundles are *sort of* like shrinkwraps, in that the branch is defined
911 // by the contents of the package. however, in their case, rather than
912 // shipping a virtual tree that must be reified, they ship an entire
913 // reified actual tree that must be unpacked and not modified.
914 [_loadBundlesAndUpdateTrees] (depth = 0, bundlesByDepth) {
915 let maxBundleDepth
916 if (!bundlesByDepth) {
917 bundlesByDepth = new Map()
918 maxBundleDepth = -1
919 dfwalk({
920 tree: this.diff,
921 visit: diff => {
922 const node = diff.ideal
923 if (!node) {
924 return
925 }
926 if (node.isProjectRoot) {
927 return
928 }
929
930 const { bundleDependencies } = node.package
931 if (bundleDependencies && bundleDependencies.length) {
932 maxBundleDepth = Math.max(maxBundleDepth, node.depth)
933 if (!bundlesByDepth.has(node.depth)) {
934 bundlesByDepth.set(node.depth, [node])
935 } else {
936 bundlesByDepth.get(node.depth).push(node)
937 }
938 }
939 },
940 getChildren: diff => diff.children,
941 })
942
943 bundlesByDepth.set('maxBundleDepth', maxBundleDepth)
944 } else {
945 maxBundleDepth = bundlesByDepth.get('maxBundleDepth')
946 }
947
948 if (depth === 0) {
949 time.start('reify:loadBundles')
950 }
951
952 if (depth > maxBundleDepth) {
953 // if we did something, then prune the tree and update the diffs
954 if (maxBundleDepth !== -1) {
955 this.#pruneBundledMetadeps(bundlesByDepth)
956 this[_diffTrees]()
957 }
958 time.end('reify:loadBundles')
959 return
960 }
961
962 // skip any that have since been removed from the tree, eg by a
963 // shallower bundle overwriting them with a bundled meta-dep.
964 const set = (bundlesByDepth.get(depth) || [])
965 .filter(node => node.root === this.idealTree &&
966 node.target !== node.root &&
967 !this[_trashList].has(node.path))
968
969 if (!set.length) {
970 return this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth)
971 }
972
973 // extract all the nodes with bundles
974 return promiseCallLimit(set.map(node => {
975 return () => {
976 this.#bundleUnpacked.add(node)
977 return this[_reifyNode](node)
978 }
979 }), { rejectLate: true })
980 // then load their unpacked children and move into the ideal tree
981 .then(nodes =>
982 promiseAllRejectLate(nodes.map(async node => {
983 const arb = new this.constructor({
984 ...this.options,
985 path: node.path,
986 })
987 const notTransplanted = new Set(node.children.keys())
988 await arb.loadActual({
989 root: node,
990 // don't transplant any sparse folders we created
991 // loadActual will set node.package to {} for empty directories
992 // if by chance there are some empty folders in the node_modules
993 // tree for some other reason, then ok, ignore those too.
994 transplantFilter: node => {
995 if (node.package._id) {
996 // it's actually in the bundle if it gets transplanted
997 notTransplanted.delete(node.name)
998 return true
999 } else {
1000 return false
1001 }
1002 },
1003 })
1004 for (const name of notTransplanted) {
1005 this.#bundleMissing.add(node.children.get(name))
1006 }
1007 })))
1008 // move onto the next level of bundled items
1009 .then(() => this[_loadBundlesAndUpdateTrees](depth + 1, bundlesByDepth))
1010 }
1011
1012 // https://github.com/npm/cli/issues/1597#issuecomment-667639545
1013 #pruneBundledMetadeps (bundlesByDepth) {
1014 const bundleShadowed = new Set()
1015
1016 // Example dep graph:
1017 // root -> (a, c)
1018 // a -> BUNDLE(b)
1019 // b -> c
1020 // c -> b
1021 //
1022 // package tree:
1023 // root
1024 // +-- a
1025 // | +-- b(1)
1026 // | +-- c(1)
1027 // +-- b(2)
1028 // +-- c(2)
1029 // 1. mark everything that's shadowed by anything in the bundle. This
1030 // marks b(2) and c(2).
1031 // 2. anything with edgesIn from outside the set, mark not-extraneous,
1032 // remove from set. This unmarks c(2).
1033 // 3. continue until no change
1034 // 4. remove everything in the set from the tree. b(2) is pruned
1035
1036 // create the list of nodes shadowed by children of bundlers
1037 for (const bundles of bundlesByDepth.values()) {
1038 // skip the 'maxBundleDepth' item
1039 if (!Array.isArray(bundles)) {
1040 continue
1041 }
1042 for (const node of bundles) {
1043 for (const name of node.children.keys()) {
1044 const shadow = node.parent.resolve(name)
1045 if (!shadow) {
1046 continue
1047 }
1048 bundleShadowed.add(shadow)
1049 shadow.extraneous = true
1050 }
1051 }
1052 }
1053
1054 // lib -> (a@1.x) BUNDLE(a@1.2.3 (b@1.2.3))
1055 // a@1.2.3 -> (b@1.2.3)
1056 // a@1.3.0 -> (b@2)
1057 // b@1.2.3 -> ()
1058 // b@2 -> (c@2)
1059 //
1060 // root
1061 // +-- lib
1062 // | +-- a@1.2.3
1063 // | +-- b@1.2.3
1064 // +-- b@2 <-- shadowed, now extraneous
1065 // +-- c@2 <-- also shadowed, because only dependent is shadowed
1066 for (const shadow of bundleShadowed) {
1067 for (const shadDep of shadow.edgesOut.values()) {
1068 /* istanbul ignore else - pretty unusual situation, just being
1069 * defensive here. Would mean that a bundled dep has a dependency
1070 * that is unmet. which, weird, but if you bundle it, we take
1071 * whatever you put there and assume the publisher knows best. */
1072 if (shadDep.to) {
1073 bundleShadowed.add(shadDep.to)
1074 shadDep.to.extraneous = true
1075 }
1076 }
1077 }
1078
1079 let changed
1080 do {
1081 changed = false
1082 for (const shadow of bundleShadowed) {
1083 for (const edge of shadow.edgesIn) {
1084 if (!bundleShadowed.has(edge.from)) {
1085 shadow.extraneous = false
1086 bundleShadowed.delete(shadow)
1087 changed = true
1088 break
1089 }
1090 }
1091 }
1092 } while (changed)
1093
1094 for (const shadow of bundleShadowed) {
1095 this[_addNodeToTrashList](shadow)
1096 shadow.root = null
1097 }
1098 }
1099
1100 async [_submitQuickAudit] () {
1101 if (this.options.audit === false) {
1102 this.auditReport = null
1103 return
1104 }
1105
1106 // we submit the quick audit at this point in the process, as soon as
1107 // we have all the deps resolved, so that it can overlap with the other
1108 // actions as much as possible. Stash the promise, which we resolve
1109 // before finishing the reify() and returning the tree. Thus, we do
1110 // NOT return the promise, as the intent is for this to run in parallel
1111 // with the reification, and be resolved at a later time.
1112 const timeEnd = time.start('reify:audit')
1113 const options = { ...this.options }
1114 const tree = this.idealTree
1115
1116 // if we're operating on a workspace, only audit the workspace deps
1117 if (this.options.workspaces.length) {
1118 options.filterSet = this.workspaceDependencySet(
1119 tree,
1120 this.options.workspaces,
1121 this.options.includeWorkspaceRoot
1122 )
1123 }
1124
1125 this.auditReport = AuditReport.load(tree, options).then(res => {
1126 timeEnd()
1127 return res
1128 })
1129 }
1130
1131 // ok! actually unpack stuff into their target locations!
1132 // The sparse tree has already been created, so we walk the diff
1133 // kicking off each unpack job. If any fail, we rm the sparse
1134 // tree entirely and try to put everything back where it was.
1135 [_unpackNewModules] () {
1136 const timeEnd = time.start('reify:unpack')
1137 const unpacks = []
1138 dfwalk({
1139 tree: this.diff,
1140 visit: diff => {
1141 // no unpacking if we don't want to change this thing
1142 if (diff.action !== 'CHANGE' && diff.action !== 'ADD') {
1143 return
1144 }
1145
1146 const node = diff.ideal
1147 const bd = this.#bundleUnpacked.has(node)
1148 const sw = this.#shrinkwrapInflated.has(node)
1149 const bundleMissing = this.#bundleMissing.has(node)
1150
1151 // check whether we still need to unpack this one.
1152 // test the inDepBundle last, since that's potentially a tree walk.
1153 const doUnpack = node && // can't unpack if removed!
1154 // root node already exists
1155 !node.isRoot &&
1156 // already unpacked to read bundle
1157 !bd &&
1158 // already unpacked to read sw
1159 !sw &&
1160 // already unpacked by another dep's bundle
1161 (bundleMissing || !node.inDepBundle)
1162
1163 if (doUnpack) {
1164 unpacks.push(this[_reifyNode](node))
1165 }
1166 },
1167 getChildren: diff => diff.children,
1168 })
1169 return promiseAllRejectLate(unpacks).then(timeEnd)
1170 }
1171
1172 // This is the part where we move back the unchanging nodes that were
1173 // the children of a node that did change. If this fails, the rollback
1174 // is a three-step process. First, we try to move the retired unchanged
1175 // nodes BACK to their retirement folders, then delete the sparse tree,
1176 // then move everything out of retirement.
1177 [_moveBackRetiredUnchanged] () {
1178 // get a list of all unchanging children of any shallow retired nodes
1179 // if they are not the ancestor of any node in the diff set, then the
1180 // directory won't already exist, so just rename it over.
1181 // This is sort of an inverse diff tree, of all the nodes where
1182 // the actualTree and idealTree _don't_ differ, starting from the
1183 // shallowest nodes that we moved aside in the first place.
1184 const timeEnd = time.start('reify:unretire')
1185 const moves = this.#retiredPaths
1186 this.#retiredUnchanged = {}
1187 return promiseAllRejectLate(this.diff.children.map(diff => {
1188 // skip if nothing was retired
1189 if (diff.action !== 'CHANGE' && diff.action !== 'REMOVE') {
1190 return
1191 }
1192
1193 const { path: realFolder } = diff.actual
1194 const retireFolder = moves[realFolder]
1195 /* istanbul ignore next - should be impossible */
1196 debug(() => {
1197 if (!retireFolder) {
1198 const er = new Error('trying to un-retire but not retired')
1199 throw Object.assign(er, {
1200 realFolder,