codekingpro/portable-devtools
114k
1// Arborist.rebuild({path = this.path}) will do all the binlinks and
2// bundle building needed. Called by reify, and by `npm rebuild`.
3
4const PackageJson = require('@npmcli/package-json')
5const binLinks = require('bin-links')
6const localeCompare = require('@isaacs/string-locale-compare')('en')
7const promiseAllRejectLate = require('promise-all-reject-late')
8const runScript = require('@npmcli/run-script')
9const { callLimit: promiseCallLimit } = require('promise-call-limit')
10const { depth: dfwalk } = require('treeverse')
11const { isNodeGypPackage, defaultGypInstallScript } = require('@npmcli/node-gyp')
12const { promiseRetry } = require('@gar/promise-retry')
13const { log, time } = require('proc-log')
14const { resolve } = require('node:path')
15
16const boolEnv = b => b ? '1' : ''
17const sortNodes = (a, b) => (a.depth - b.depth) || localeCompare(a.path, b.path)
18
19const _checkBins = Symbol.for('checkBins')
20
21// defined by reify mixin
22const _handleOptionalFailure = Symbol.for('handleOptionalFailure')
23const _trashList = Symbol.for('trashList')
24
25module.exports = cls => class Builder extends cls {
26 #doHandleOptionalFailure
27 #oldMeta = null
28 #queues = {
29 preinstall: [],
30 install: [],
31 postinstall: [],
32 prepare: [],
33 bin: [],
34 }
35
36 async rebuild ({ nodes, handleOptionalFailure = false } = {}) {
37 // nothing to do if we're not building anything!
38 if (this.options.ignoreScripts && !this.options.binLinks) {
39 return
40 }
41
42 // when building for the first time, as part of reify, we ignore
43 // failures in optional nodes, and just delete them. however, when
44 // running JUST a rebuild, we treat optional failures as real fails
45 this.#doHandleOptionalFailure = handleOptionalFailure
46
47 if (!nodes) {
48 nodes = await this.#loadDefaultNodes()
49 }
50
51 // separates links nodes so that it can run
52 // prepare scripts and link bins in the expected order
53 const timeEnd = time.start('build')
54
55 const {
56 depNodes,
57 linkNodes,
58 } = this.#retrieveNodesByType(nodes)
59
60 // build regular deps
61 await this.#build(depNodes, {})
62
63 // build link deps
64 if (linkNodes.size) {
65 this.#queues = {
66 preinstall: [],
67 install: [],
68 postinstall: [],
69 prepare: [],
70 bin: [],
71 }
72 await this.#build(linkNodes, { type: 'links' })
73 }
74
75 timeEnd()
76 }
77
78 // if we don't have a set of nodes, then just rebuild
79 // the actual tree on disk.
80 async #loadDefaultNodes () {
81 let nodes
82 const tree = await this.loadActual()
83 let filterSet
84 if (!this.options.workspacesEnabled) {
85 filterSet = this.excludeWorkspacesDependencySet(tree)
86 nodes = tree.inventory.filter(node =>
87 filterSet.has(node) || node.isProjectRoot
88 )
89 } else if (this.options.workspaces.length) {
90 filterSet = this.workspaceDependencySet(
91 tree,
92 this.options.workspaces,
93 this.options.includeWorkspaceRoot
94 )
95 nodes = tree.inventory.filter(node => filterSet.has(node))
96 } else {
97 nodes = tree.inventory.values()
98 }
99 return nodes
100 }
101
102 #retrieveNodesByType (nodes) {
103 const depNodes = new Set()
104 const linkNodes = new Set()
105 const storeNodes = new Set()
106
107 for (const node of nodes) {
108 if (node.isStoreLink) {
109 storeNodes.add(node)
110 } else if (node.isLink) {
111 linkNodes.add(node)
112 } else {
113 depNodes.add(node)
114 }
115 }
116 // Make sure that store linked nodes are processed last.
117 // We can't process store links separately or else lifecycle scripts on
118 // standard nodes might not have bin links yet.
119 for (const node of storeNodes) {
120 depNodes.add(node)
121 }
122
123 // deduplicates link nodes and their targets, avoids
124 // calling lifecycle scripts twice when running `npm rebuild`
125 // ref: https://github.com/npm/cli/issues/2905
126 //
127 // we avoid doing so if global=true since `bin-links` relies
128 // on having the target nodes available in global mode.
129 if (!this.options.global) {
130 for (const node of linkNodes) {
131 depNodes.delete(node.target)
132 }
133 }
134
135 return {
136 depNodes,
137 linkNodes,
138 }
139 }
140
141 async #build (nodes, { type = 'deps' }) {
142 const timeEnd = time.start(`build:${type}`)
143
144 await this.#buildQueues(nodes)
145
146 if (!this.options.ignoreScripts) {
147 await this.#runScripts('preinstall')
148 }
149
150 // links should run prepare scripts and only link bins after that
151 if (type === 'links') {
152 if (!this.options.ignoreScripts) {
153 await this.#runScripts('prepare')
154 }
155 }
156 if (this.options.binLinks) {
157 await this.#linkAllBins()
158 }
159
160 if (!this.options.ignoreScripts) {
161 await this.#runScripts('install')
162 await this.#runScripts('postinstall')
163 }
164
165 timeEnd()
166 }
167
168 async #buildQueues (nodes) {
169 const timeEnd = time.start('build:queue')
170 const set = new Set()
171
172 const promises = []
173 for (const node of nodes) {
174 promises.push(this.#addToBuildSet(node, set))
175
176 // if it has bundle deps, add those too, if rebuildBundle
177 if (this.options.rebuildBundle !== false) {
178 const bd = node.package.bundleDependencies
179 if (bd && bd.length) {
180 dfwalk({
181 tree: node,
182 leave: node => promises.push(this.#addToBuildSet(node, set)),
183 getChildren: node => [...node.children.values()],
184 filter: node => node.inBundle,
185 })
186 }
187 }
188 }
189 await promiseAllRejectLate(promises)
190
191 // now sort into the queues for the 4 things we have to do
192 // run in the same predictable order that buildIdealTree uses
193 // there's no particular reason for doing it in this order rather
194 // than another, but sorting *somehow* makes it consistent.
195 const queue = [...set].sort(sortNodes)
196
197 for (const node of queue) {
198 const { package: { bin, scripts = {} } } = node.target
199 const { preinstall, install, postinstall, prepare } = scripts
200 const tests = { bin, preinstall, install, postinstall, prepare }
201 for (const [key, has] of Object.entries(tests)) {
202 if (has) {
203 this.#queues[key].push(node)
204 }
205 }
206 }
207 timeEnd()
208 }
209
210 async [_checkBins] (node) {
211 // if the node is a global top, and we're not in force mode, then
212 // any existing bins need to either be missing, or a symlink into
213 // the node path. Otherwise a package can have a preinstall script
214 // that unlinks something, to allow them to silently overwrite system
215 // binaries, which is unsafe and insecure.
216 if (!node.globalTop || this.options.force) {
217 return
218 }
219 const { path, package: pkg } = node
220 await binLinks.checkBins({ pkg, path, top: true, global: true })
221 }
222
223 async #addToBuildSet (node, set, refreshed = false) {
224 if (set.has(node)) {
225 return
226 }
227
228 if (this.#oldMeta === null) {
229 const { root: { meta } } = node
230 this.#oldMeta = meta && meta.loadedFromDisk &&
231 !(meta.originalLockfileVersion >= 2)
232 }
233
234 const { package: pkg, hasInstallScript } = node.target
235 const { gypfile, bin, scripts = {} } = pkg
236
237 const { preinstall, install, postinstall, prepare } = scripts
238 const anyScript = preinstall || install || postinstall || prepare
239 if (!refreshed && !anyScript && (hasInstallScript || this.#oldMeta)) {
240 // we either have an old metadata (and thus might have scripts)
241 // or we have an indication that there's install scripts (but
242 // don't yet know what they are) so we have to load the package.json
243 // from disk to see what the deal is. Failure here just means
244 // no scripts to add, probably borked package.json.
245 // add to the set then remove while we're reading the pj, so we
246 // don't accidentally hit it multiple times.
247 set.add(node)
248 const { content: pkg } = await PackageJson.normalize(node.path).catch(() => {
249 return { content: {} }
250 })
251 set.delete(node)
252
253 const { scripts = {} } = pkg
254 node.package.scripts = scripts
255 return this.#addToBuildSet(node, set, true)
256 }
257
258 // Rebuild node-gyp dependencies lacking an install or preinstall script
259 // note that 'scripts' might be missing entirely, and the package may
260 // set gypfile:false to avoid this automatic detection.
261 const isGyp = gypfile !== false &&
262 !install &&
263 !preinstall &&
264 await isNodeGypPackage(node.path)
265
266 if (bin || preinstall || install || postinstall || prepare || isGyp) {
267 if (bin) {
268 await this[_checkBins](node)
269 }
270 if (isGyp) {
271 scripts.install = defaultGypInstallScript
272 node.package.scripts = scripts
273 }
274 set.add(node)
275 }
276 }
277
278 async #runScripts (event) {
279 const queue = this.#queues[event]
280
281 if (!queue.length) {
282 return
283 }
284
285 const timeEnd = time.start(`build:run:${event}`)
286 const stdio = this.options.foregroundScripts ? 'inherit' : 'pipe'
287 const limit = this.options.foregroundScripts ? 1 : undefined
288 await promiseCallLimit(queue.map(node => async () => {
289 const {
290 path,
291 integrity,
292 resolved,
293 optional,
294 peer,
295 dev,
296 devOptional,
297 package: pkg,
298 location,
299 } = node.target
300
301 // skip any that we know we'll be deleting
302 // or links to store entries (their scripts run on the store
303 // entry itself, not through the link)
304 if (this[_trashList].has(path) || (node.isLink && node.target?.isInStore)) {
305 return
306 }
307
308 const timeEndLocation = time.start(`build:run:${event}:${location}`)
309 log.info('run', pkg._id, event, location, pkg.scripts[event])
310 const env = {
311 npm_package_resolved: resolved,
312 npm_package_integrity: integrity,
313 npm_package_json: resolve(path, 'package.json'),
314 npm_package_optional: boolEnv(optional),
315 npm_package_dev: boolEnv(dev),
316 npm_package_peer: boolEnv(peer),
317 npm_package_dev_optional:
318 boolEnv(devOptional && !dev && !optional),
319 }
320 const runOpts = {
321 event,
322 path,
323 pkg,
324 stdio,
325 env,
326 scriptShell: this.options.scriptShell,
327 }
328 const p = runScript(runOpts).catch(er => {
329 const { code, signal } = er
330 log.info('run', pkg._id, event, { code, signal })
331 throw er
332 }).then(({ args, code, signal, stdout, stderr }) => {
333 this.scriptsRun.add({
334 pkg,
335 path,
336 event,
337 // I do not know why this needs to be on THIS line but refactoring
338 // this function would be quite a process
339 // eslint-disable-next-line promise/always-return
340 cmd: args && args[args.length - 1],
341 env,
342 code,
343 signal,
344 stdout,
345 stderr,
346 })
347 log.info('run', pkg._id, event, { code, signal })
348 })
349
350 await (this.#doHandleOptionalFailure
351 ? this[_handleOptionalFailure](node, p)
352 : p)
353
354 timeEndLocation()
355 }), { limit })
356 timeEnd()
357 }
358
359 async #linkAllBins () {
360 const queue = this.#queues.bin
361 if (!queue.length) {
362 return
363 }
364
365 const timeEnd = time.start('build:link')
366 const promises = []
367 // sort the queue by node path, so that the module-local collision
368 // detector in bin-links will always resolve the same way.
369 for (const node of queue.sort(sortNodes)) {
370 // TODO these run before they're awaited
371 promises.push(this.#createBinLinks(node))
372 }
373
374 await promiseAllRejectLate(promises)
375 timeEnd()
376 }
377
378 async #createBinLinks (node) {
379 if (this[_trashList].has(node.path)) {
380 return
381 }
382
383 const timeEnd = time.start(`build:link:${node.location}`)
384
385 // On Windows, antivirus/indexer can transiently lock files, causing EPERM/EACCES/EBUSY on the rename inside write-file-atomic (used by bin-links/fix-bin.js), so, retry with backoff.
386 const p = promiseRetry((retry) => binLinks({
387 pkg: node.package,
388 path: node.path,
389 top: !!(node.isTop || node.globalTop),
390 force: this.options.force,
391 global: !!node.globalTop,
392 }).catch(/* istanbul ignore next - Windows-only transient antivirus locks */ err => {
393 if (process.platform === 'win32' &&
394 (err.code === 'EPERM' || err.code === 'EACCES' || err.code === 'EBUSY')) {
395 return retry(err)
396 }
397 throw err
398 }), { retries: 5, minTimeout: 500 })
399
400 await (this.#doHandleOptionalFailure
401 ? this[_handleOptionalFailure](node, p)
402 : p)
403
404 timeEnd()
405 }
406}
407