codekingpro/portable-devtools
115k
1// This is the base class that the other fetcher types in lib
2// all descend from.
3// It handles the unpacking and retry logic that is shared among
4// all of the other Fetcher types.
5
6const { basename, dirname } = require('node:path')
7const { rm, mkdir } = require('node:fs/promises')
8const PackageJson = require('@npmcli/package-json')
9const cacache = require('cacache')
10const fsm = require('fs-minipass')
11const getContents = require('@npmcli/installed-package-contents')
12const npa = require('npm-package-arg')
13const { promiseRetry } = require('@gar/promise-retry')
14const ssri = require('ssri')
15const tar = require('tar')
16const { Minipass } = require('minipass')
17const { log } = require('proc-log')
18const _ = require('./util/protected.js')
19const cacheDir = require('./util/cache-dir.js')
20const isPackageBin = require('./util/is-package-bin.js')
21const removeTrailingSlashes = require('./util/trailing-slashes.js')
22
23// Pacote is only concerned with the package.json contents
24const packageJsonPrepare = (p) => PackageJson.prepare(p).then(pkg => pkg.content)
25const packageJsonNormalize = (p) => PackageJson.normalize(p).then(pkg => pkg.content)
26
27class FetcherBase {
28 constructor (spec, opts) {
29 if (!opts || typeof opts !== 'object') {
30 throw new TypeError('options object is required')
31 }
32 this.spec = npa(spec, opts.where)
33
34 this.allowGitIgnore = !!opts.allowGitIgnore
35
36 // a bit redundant because presumably the caller already knows this,
37 // but it makes it easier to not have to keep track of the requested
38 // spec when we're dispatching thousands of these at once, and normalizing
39 // is nice. saveSpec is preferred if set, because it turns stuff like
40 // x/y#committish into github:x/y#committish. use name@rawSpec for
41 // registry deps so that we turn xyz and xyz@ -> xyz@
42 this.from = this.spec.registry
43 ? `${this.spec.name}@${this.spec.rawSpec}` : this.spec.saveSpec
44
45 this.#assertType()
46 // clone the opts object so that others aren't upset when we mutate it
47 // by adding/modifying the integrity value.
48 this.opts = { ...opts }
49
50 this.cache = opts.cache || cacheDir().cacache
51 this.tufCache = opts.tufCache || cacheDir().tufcache
52 this.resolved = opts.resolved || null
53
54 // default to caching/verifying with sha512, that's what we usually have
55 // need to change this default, or start overriding it, when sha512
56 // is no longer strong enough.
57 this.defaultIntegrityAlgorithm = opts.defaultIntegrityAlgorithm || 'sha512'
58
59 if (typeof opts.integrity === 'string') {
60 this.opts.integrity = ssri.parse(opts.integrity)
61 }
62
63 this.package = null
64 this.type = this.constructor.name
65 this.fmode = opts.fmode || 0o666
66 this.dmode = opts.dmode || 0o777
67 // we don't need a default umask, because we don't chmod files coming
68 // out of package tarballs. they're forced to have a mode that is
69 // valid, regardless of what's in the tarball entry, and then we let
70 // the process's umask setting do its job. but if configured, we do
71 // respect it.
72 this.umask = opts.umask || 0
73
74 this.preferOnline = !!opts.preferOnline
75 this.preferOffline = !!opts.preferOffline
76 this.offline = !!opts.offline
77
78 this.before = opts.before
79 this.fullMetadata = this.before ? true : !!opts.fullMetadata
80 this.fullReadJson = !!opts.fullReadJson
81 this[_.readPackageJson] = this.fullReadJson
82 ? packageJsonPrepare
83 : packageJsonNormalize
84
85 // rrh is a registry hostname or 'never' or 'always'
86 // defaults to registry.npmjs.org
87 this.replaceRegistryHost = (!opts.replaceRegistryHost || opts.replaceRegistryHost === 'npmjs') ?
88 'registry.npmjs.org' : opts.replaceRegistryHost
89
90 this.defaultTag = opts.defaultTag || 'latest'
91 this.registry = removeTrailingSlashes(opts.registry || 'https://registry.npmjs.org')
92
93 // command to run 'prepare' scripts on directories and git dirs
94 // To use pacote with yarn, for example, set npmBin to 'yarn'
95 // and npmCliConfig with yarn's equivalents.
96 this.npmBin = opts.npmBin || 'npm'
97
98 // command to install deps for preparing
99 this.npmInstallCmd = opts.npmInstallCmd || ['install', '--force']
100
101 // XXX fill more of this in based on what we know from this.opts
102 // we explicitly DO NOT fill in --tag, though, since we are often
103 // going to be packing in the context of a publish, which may set
104 // a dist-tag, but certainly wants to keep defaulting to latest.
105 this.npmCliConfig = opts.npmCliConfig || [
106 `--cache=${dirname(this.cache)}`,
107 `--prefer-offline=${!!this.preferOffline}`,
108 `--prefer-online=${!!this.preferOnline}`,
109 `--offline=${!!this.offline}`,
110 ...(this.before ? [`--before=${this.before.toISOString()}`] : []),
111 '--no-progress',
112 '--no-save',
113 '--no-audit',
114 // override any omit settings from the environment
115 '--include=dev',
116 '--include=peer',
117 '--include=optional',
118 // we need the actual things, not just the lockfile
119 '--no-package-lock-only',
120 '--no-dry-run',
121 ]
122 }
123
124 get integrity () {
125 return this.opts.integrity || null
126 }
127
128 set integrity (i) {
129 if (!i) {
130 return
131 }
132
133 i = ssri.parse(i)
134 const current = this.opts.integrity
135
136 // do not ever update an existing hash value, but do
137 // merge in NEW algos and hashes that we don't already have.
138 if (current) {
139 current.merge(i)
140 } else {
141 this.opts.integrity = i
142 }
143 }
144
145 get notImplementedError () {
146 return new Error('not implemented in this fetcher type: ' + this.type)
147 }
148
149 // override in child classes
150 // Returns a Promise that resolves to this.resolved string value
151 resolve () {
152 return this.resolved ? Promise.resolve(this.resolved)
153 : Promise.reject(this.notImplementedError)
154 }
155
156 packument () {
157 return Promise.reject(this.notImplementedError)
158 }
159
160 // override in child class
161 // returns a manifest containing:
162 // - name
163 // - version
164 // - _resolved
165 // - _integrity
166 // - plus whatever else was in there (corgi, full metadata, or pj file)
167 manifest () {
168 return Promise.reject(this.notImplementedError)
169 }
170
171 // private, should be overridden.
172 // Note that they should *not* calculate or check integrity or cache,
173 // but *just* return the raw tarball data stream.
174 [_.tarballFromResolved] () {
175 throw this.notImplementedError
176 }
177
178 // public, should not be overridden
179 tarball () {
180 return this.tarballStream(stream => stream.concat().then(data => {
181 data.integrity = this.integrity && String(this.integrity)
182 data.resolved = this.resolved
183 data.from = this.from
184 return data
185 }))
186 }
187
188 // private
189 // Note: cacache will raise a EINTEGRITY error if the integrity doesn't match
190 #tarballFromCache () {
191 const startTime = Date.now()
192 const stream = cacache.get.stream.byDigest(this.cache, this.integrity, this.opts)
193 const elapsedTime = Date.now() - startTime
194 // cache is good, so log it as a hit in particular since there was no fetch logged
195 log.http(
196 'cache',
197 `${this.spec} ${elapsedTime}ms (cache hit)`
198 )
199 return stream
200 }
201
202 get [_.cacheFetches] () {
203 return true
204 }
205
206 #istream (stream) {
207 // if not caching this, just return it
208 if (!this.opts.cache || !this[_.cacheFetches]) {
209 // instead of creating a new integrity stream, we only piggyback on the
210 // provided stream's events
211 if (stream.hasIntegrityEmitter) {
212 stream.on('integrity', i => this.integrity = i)
213 return stream
214 }
215
216 const istream = ssri.integrityStream(this.opts)
217 istream.on('integrity', i => this.integrity = i)
218 stream.on('error', err => istream.emit('error', err))
219 return stream.pipe(istream)
220 }
221
222 // we have to return a stream that gets ALL the data, and proxies errors,
223 // but then pipe from the original tarball stream into the cache as well.
224 // To do this without losing any data, and since the cacache put stream
225 // is not a passthrough, we have to pipe from the original stream into
226 // the cache AFTER we pipe into the middleStream. Since the cache stream
227 // has an asynchronous flush to write its contents to disk, we need to
228 // defer the middleStream end until the cache stream ends.
229 const middleStream = new Minipass()
230 stream.on('error', err => middleStream.emit('error', err))
231 stream.pipe(middleStream, { end: false })
232 const cstream = cacache.put.stream(
233 this.opts.cache,
234 `pacote:tarball:${this.from}`,
235 this.opts
236 )
237 cstream.on('integrity', i => this.integrity = i)
238 cstream.on('error', err => stream.emit('error', err))
239 stream.pipe(cstream)
240
241 // eslint-disable-next-line promise/catch-or-return
242 cstream.promise().catch(() => {}).then(() => middleStream.end())
243 return middleStream
244 }
245
246 pickIntegrityAlgorithm () {
247 return this.integrity ? this.integrity.pickAlgorithm(this.opts)
248 : this.defaultIntegrityAlgorithm
249 }
250
251 // TODO: check error class, once those are rolled out to our deps
252 isDataCorruptionError (er) {
253 return er.code === 'EINTEGRITY' || er.code === 'Z_DATA_ERROR'
254 }
255
256 // override the types getter
257 get types () {
258 return false
259 }
260
261 #assertType () {
262 if (this.types && !this.types.includes(this.spec.type)) {
263 throw new TypeError(`Wrong spec type (${
264 this.spec.type
265 }) for ${
266 this.constructor.name
267 }. Supported types: ${this.types.join(', ')}`)
268 }
269 }
270
271 // We allow ENOENTs from cacache, but not anywhere else.
272 // An ENOENT trying to read a tgz file, for example, is Right Out.
273 isRetriableError (er) {
274 // TODO: check error class, once those are rolled out to our deps
275 return this.isDataCorruptionError(er) ||
276 er.code === 'ENOENT' ||
277 er.code === 'EISDIR'
278 }
279
280 // Mostly internal, but has some uses
281 // Pass in a function which returns a promise
282 // Function will be called 1 or more times with streams that may fail.
283 // Retries:
284 // Function MUST handle errors on the stream by rejecting the promise,
285 // so that retry logic can pick it up and either retry or fail whatever
286 // promise it was making (ie, failing extraction, etc.)
287 //
288 // The return value of this method is a Promise that resolves the same
289 // as whatever the streamHandler resolves to.
290 //
291 // This should never be overridden by child classes, but it is public.
292 tarballStream (streamHandler) {
293 // Only short-circuit via cache if we have everything else we'll need,
294 // and the user has not expressed a preference for checking online.
295
296 const fromCache = (
297 !this.preferOnline &&
298 this.integrity &&
299 this.resolved
300 ) ? streamHandler(this.#tarballFromCache()).catch(er => {
301 if (this.isDataCorruptionError(er)) {
302 log.warn('tarball', `cached data for ${
303 this.spec
304 } (${this.integrity}) seems to be corrupted. Refreshing cache.`)
305 return this.cleanupCached().then(() => {
306 throw er
307 })
308 } else {
309 throw er
310 }
311 }) : null
312
313 const fromResolved = er => {
314 if (er) {
315 if (!this.isRetriableError(er)) {
316 throw er
317 }
318 log.silly('tarball', `no local data for ${
319 this.spec
320 }. Extracting by manifest.`)
321 }
322 return this.resolve().then(() => promiseRetry(tryAgain =>
323 streamHandler(this.#istream(this[_.tarballFromResolved]()))
324 .catch(streamErr => {
325 // Most likely data integrity. A cache ENOENT error is unlikely
326 // here, since we're definitely not reading from the cache, but it
327 // IS possible that the fetch subsystem accessed the cache, and the
328 // entry got blown away or something. Try one more time to be sure.
329 if (this.isRetriableError(streamErr)) {
330 log.warn('tarball', `tarball data for ${
331 this.spec
332 } (${this.integrity}) seems to be corrupted. Trying again.`)
333 return this.cleanupCached().then(() => tryAgain(streamErr))
334 }
335 throw streamErr
336 }), { retries: 1, minTimeout: 0, maxTimeout: 0 }))
337 }
338
339 return fromCache ? fromCache.catch(fromResolved) : fromResolved()
340 }
341
342 cleanupCached () {
343 return cacache.rm.content(this.cache, this.integrity, this.opts)
344 }
345
346 #empty (path) {
347 return getContents({ path, depth: 1 }).then(contents => Promise.all(
348 contents.map(entry => rm(entry, { recursive: true, force: true }))))
349 }
350
351 async #mkdir (dest) {
352 await this.#empty(dest)
353 return await mkdir(dest, { recursive: true })
354 }
355
356 // extraction is always the same. the only difference is where
357 // the tarball comes from.
358 async extract (dest) {
359 await this.#mkdir(dest)
360 return this.tarballStream((tarball) => this.#extract(dest, tarball))
361 }
362
363 #toFile (dest) {
364 return this.tarballStream(str => new Promise((res, rej) => {
365 const writer = new fsm.WriteStream(dest)
366 str.on('error', er => writer.emit('error', er))
367 writer.on('error', er => rej(er))
368 writer.on('close', () => res({
369 integrity: this.integrity && String(this.integrity),
370 resolved: this.resolved,
371 from: this.from,
372 }))
373 str.pipe(writer)
374 }))
375 }
376
377 // don't use this.#mkdir because we don't want to rimraf anything
378 async tarballFile (dest) {
379 const dir = dirname(dest)
380 await mkdir(dir, { recursive: true })
381 return this.#toFile(dest)
382 }
383
384 #extract (dest, tarball) {
385 const extractor = tar.x(this.#tarxOptions({ cwd: dest }))
386 const p = new Promise((resolve, reject) => {
387 extractor.on('end', () => {
388 resolve({
389 resolved: this.resolved,
390 integrity: this.integrity && String(this.integrity),
391 from: this.from,
392 })
393 })
394
395 extractor.on('error', er => {
396 log.warn('tar', er.message)
397 log.silly('tar', er)
398 reject(er)
399 })
400
401 tarball.on('error', er => reject(er))
402 })
403
404 tarball.pipe(extractor)
405 return p
406 }
407
408 // always ensure that entries are at least as permissive as our configured
409 // dmode/fmode, but never more permissive than the umask allows.
410 #entryMode (path, mode, type) {
411 const m = /Directory|GNUDumpDir/.test(type) ? this.dmode
412 : /File$/.test(type) ? this.fmode
413 : /* istanbul ignore next - should never happen in a pkg */ 0
414
415 // make sure package bins are executable
416 const exe = isPackageBin(this.package, path) ? 0o111 : 0
417 // always ensure that files are read/writable by the owner
418 return ((mode | m) & ~this.umask) | exe | 0o600
419 }
420
421 #tarxOptions ({ cwd }) {
422 const sawIgnores = new Set()
423 return {
424 cwd,
425 noChmod: true,
426 noMtime: true,
427 filter: (name, entry) => {
428 if (/Link$/.test(entry.type)) {
429 return false
430 }
431 entry.mode = this.#entryMode(entry.path, entry.mode, entry.type)
432 // this replicates the npm pack behavior where .gitignore files
433 // are treated like .npmignore files, but only if a .npmignore
434 // file is not present.
435 if (/File$/.test(entry.type)) {
436 const base = basename(entry.path)
437 if (base === '.npmignore') {
438 sawIgnores.add(entry.path)
439 } else if (base === '.gitignore' && !this.allowGitIgnore) {
440 // rename, but only if there's not already a .npmignore
441 const ni = entry.path.replace(/\.gitignore$/, '.npmignore')
442 if (sawIgnores.has(ni)) {
443 return false
444 }
445 entry.path = ni
446 }
447 return true
448 }
449 },
450 strip: 1,
451 onwarn: /* istanbul ignore next - we can trust that tar logs */
452 (code, msg, data) => {
453 log.warn('tar', code, msg)
454 log.silly('tar', code, msg, data)
455 },
456 umask: this.umask,
457 // always ignore ownership info from tarball metadata
458 preserveOwner: false,
459 }
460 }
461}
462
463module.exports = FetcherBase
464
465// Child classes
466const GitFetcher = require('./git.js')
467const RegistryFetcher = require('./registry.js')
468const FileFetcher = require('./file.js')
469const DirFetcher = require('./dir.js')
470const RemoteFetcher = require('./remote.js')
471
472// possible values for allow: 'all', 'root', 'none'
473const canUse = ({ allow = 'all', isRoot = false, allowType, spec }) => {
474 if (allow === 'all') {
475 return true
476 }
477 if (allow !== 'none' && isRoot) {
478 return true
479 }
480 throw Object.assign(
481 new Error(`Fetching${allow === 'root' ? ' non-root' : ''} packages of type "${allowType}" have been disabled`),
482 {
483 code: `EALLOW${allowType.toUpperCase()}`,
484 package: spec.toString(),
485 }
486 )
487}
488
489// Get an appropriate fetcher object from a spec and options
490FetcherBase.get = (rawSpec, opts = {}) => {
491 const spec = npa(rawSpec, opts.where)
492 switch (spec.type) {
493 case 'git':
494 canUse({ allow: opts.allowGit, isRoot: opts._isRoot, allowType: 'git', spec })
495 return new GitFetcher(spec, opts)
496
497 case 'remote':
498 canUse({ allow: opts.allowRemote, isRoot: opts._isRoot, allowType: 'remote', spec })
499 return new RemoteFetcher(spec, opts)
500
501 case 'version':
502 case 'range':
503 case 'tag':
504 case 'alias':
505 canUse({ allow: opts.allowRegistry, isRoot: opts._isRoot, allowType: 'registry', spec })
506 return new RegistryFetcher(spec.subSpec || spec, opts)
507
508 case 'file':
509 canUse({ allow: opts.allowFile, isRoot: opts._isRoot, allowType: 'file', spec })
510 return new FileFetcher(spec, opts)
511
512 case 'directory':
513 canUse({ allow: opts.allowDirectory, isRoot: opts._isRoot, allowType: 'directory', spec })
514 return new DirFetcher(spec, opts)
515
516 default:
517 throw new TypeError('Unknown spec type: ' + spec.type)
518 }
519}
520