codekingpro/portable-devtools
114k
1const fs = require('node:fs/promises')
2const { join } = require('node:path')
3const cacache = require('cacache')
4const pacote = require('pacote')
5const semver = require('semver')
6const npa = require('npm-package-arg')
7const jsonParse = require('json-parse-even-better-errors')
8const localeCompare = require('@isaacs/string-locale-compare')('en')
9const { log, output } = require('proc-log')
10const PkgJson = require('@npmcli/package-json')
11const abbrev = require('abbrev')
12const BaseCommand = require('../base-cmd.js')
13
14const searchCachePackage = async (path, parsed, cacheKeys) => {
15 const searchMFH = new RegExp(`^make-fetch-happen:request-cache:.*(?<!/[@a-zA-Z]+)/${parsed.name}/-/(${parsed.name}[^/]+.tgz)$`)
16 const searchPack = new RegExp(`^make-fetch-happen:request-cache:.*/${parsed.escapedName}$`)
17 const results = new Set()
18 cacheKeys = new Set(cacheKeys)
19 for (const key of cacheKeys) {
20 // match on the public key registry url format
21 if (searchMFH.test(key)) {
22 // extract the version from the filename
23 const filename = key.match(searchMFH)[1]
24 const noExt = filename.slice(0, -4)
25 const noScope = `${parsed.name.split('/').pop()}-`
26 const ver = noExt.slice(noScope.length)
27 if (semver.satisfies(ver, parsed.rawSpec)) {
28 results.add(key)
29 }
30 continue
31 }
32 // is this key a packument?
33 if (!searchPack.test(key)) {
34 continue
35 }
36
37 results.add(key)
38 let packument, details
39 try {
40 details = await cacache.get(path, key)
41 packument = jsonParse(details.data)
42 } catch {
43 // if we couldn't parse the packument, abort
44 continue
45 }
46 if (!packument.versions || typeof packument.versions !== 'object') {
47 continue
48 }
49
50 // assuming this is a packument
51 for (const ver of Object.keys(packument.versions)) {
52 if (semver.satisfies(ver, parsed.rawSpec)) {
53 if (packument.versions[ver].dist &&
54 typeof packument.versions[ver].dist === 'object' &&
55 packument.versions[ver].dist.tarball !== undefined &&
56 cacheKeys.has(`make-fetch-happen:request-cache:${packument.versions[ver].dist.tarball}`)
57 ) {
58 results.add(`make-fetch-happen:request-cache:${packument.versions[ver].dist.tarball}`)
59 }
60 }
61 }
62 }
63 return results
64}
65
66class Cache extends BaseCommand {
67 static description = 'Manipulates packages and npx cache'
68 static name = 'cache'
69 static params = ['cache']
70 static usage = [
71 'add <package-spec>',
72 'clean [<key>]',
73 'ls [<name>@<version>]',
74 'verify',
75 'npx ls',
76 'npx rm [<key>...]',
77 'npx info <key>...',
78 ]
79
80 static async completion (opts) {
81 const argv = opts.conf.argv.remain
82 if (argv.length === 2) {
83 return ['add', 'clean', 'verify', 'ls', 'npx']
84 }
85
86 // TODO - eventually...
87 switch (argv[2]) {
88 case 'verify':
89 case 'clean':
90 case 'add':
91 case 'ls':
92 return []
93 }
94 }
95
96 async exec (args) {
97 const cmd = args.shift()
98 switch (cmd) {
99 case 'rm': case 'clear': case 'clean':
100 return await this.clean(args)
101 case 'add':
102 return await this.add(args)
103 case 'verify': case 'check':
104 return await this.verify()
105 case 'ls':
106 return await this.ls(args)
107 case 'npx':
108 return await this.npx(args)
109 default:
110 throw this.usageError()
111 }
112 }
113
114 // npm cache npx
115 async npx ([cmd, ...keys]) {
116 switch (cmd) {
117 case 'ls':
118 return await this.npxLs(keys)
119 case 'rm':
120 return await this.npxRm(keys)
121 case 'info':
122 return await this.npxInfo(keys)
123 default:
124 throw this.usageError()
125 }
126 }
127
128 // npm cache clean [spec]*
129 async clean (args) {
130 // this is a derived value
131 const cachePath = this.npm.flatOptions.cache
132 if (args.length === 0) {
133 if (!this.npm.config.get('force')) {
134 throw new Error(`As of npm@5, the npm cache self-heals from corruption issues by treating integrity mismatches as cache misses.
135As a result, data extracted from the cache is guaranteed to be valid.
136If you want to make sure everything is consistent, use \`npm cache verify\` instead.
137Deleting the cache can only make npm go slower, and is not likely to correct any problems you may be encountering!
138
139On the other hand, if you're debugging an issue with the installer, or race conditions that depend on the timing of writing to an empty cache, you can use \`npm install --cache /tmp/empty-cache\` to use a temporary cache instead of removing the actual one.
140
141If you're sure you want to delete the entire cache, rerun this command with --force.`)
142 }
143 return fs.rm(cachePath, { recursive: true, force: true })
144 }
145 for (const key of args) {
146 let entry
147 try {
148 entry = await cacache.get(cachePath, key)
149 } catch {
150 log.warn('cache', `Not Found: ${key}`)
151 break
152 }
153 output.standard(`Deleted: ${key}`)
154 await cacache.rm.entry(cachePath, key)
155 // XXX this could leave other entries without content!
156 await cacache.rm.content(cachePath, entry.integrity)
157 }
158 }
159
160 // npm cache add <tarball-url>...
161 // npm cache add <pkg> <ver>...
162 // npm cache add <tarball>...
163 // npm cache add <folder>...
164 async add (args) {
165 log.silly('cache add', 'args', args)
166 if (args.length === 0) {
167 throw this.usageError('First argument to `add` is required')
168 }
169
170 await Promise.all(args.map(async spec => {
171 log.silly('cache add', 'spec', spec)
172 // we ask pacote for the thing, and then just throw the data away so that it tee-pipes it into the cache like it does for a normal request.
173 await pacote.tarball.stream(spec, stream => {
174 stream.resume()
175 return stream.promise()
176 }, { ...this.npm.flatOptions, _isRoot: true })
177
178 await pacote.manifest(spec, {
179 ...this.npm.flatOptions,
180 fullMetadata: true,
181 _isRoot: true,
182 })
183 }))
184 }
185
186 async verify () {
187 // this is a derived value
188 const cachePath = this.npm.flatOptions.cache
189 const prefix = cachePath.indexOf(process.env.HOME) === 0
190 ? `~${cachePath.slice(process.env.HOME.length)}`
191 : cachePath
192 const stats = await cacache.verify(cachePath)
193 output.standard(`Cache verified and compressed (${prefix})`)
194 output.standard(`Content verified: ${stats.verifiedContent} (${stats.keptSize} bytes)`)
195 if (stats.badContentCount) {
196 output.standard(`Corrupted content removed: ${stats.badContentCount}`)
197 }
198 if (stats.reclaimedCount) {
199 output.standard(`Content garbage-collected: ${stats.reclaimedCount} (${stats.reclaimedSize} bytes)`)
200 }
201 if (stats.missingContent) {
202 output.standard(`Missing content: ${stats.missingContent}`)
203 }
204 output.standard(`Index entries: ${stats.totalEntries}`)
205 output.standard(`Finished in ${stats.runTime.total / 1000}s`)
206 }
207
208 // npm cache ls [<spec> ...]
209 async ls (specs) {
210 // This is a derived value
211 const { cache: cachePath } = this.npm.flatOptions
212 const cacheKeys = Object.keys(await cacache.ls(cachePath))
213 if (specs.length > 0) {
214 // get results for each package spec specified
215 const results = new Set()
216 for (const spec of specs) {
217 const parsed = npa(spec)
218 if (parsed.rawSpec !== '' && parsed.type === 'tag') {
219 throw this.usageError('Cannot list cache keys for a tagged package.')
220 }
221 const keySet = await searchCachePackage(cachePath, parsed, cacheKeys)
222 for (const key of keySet) {
223 results.add(key)
224 }
225 }
226 [...results].sort(localeCompare).forEach(key => output.standard(key))
227 return
228 }
229 cacheKeys.sort(localeCompare).forEach(key => output.standard(key))
230 }
231
232 async #npxCache (keys = []) {
233 // This is a derived value
234 const { npxCache } = this.npm.flatOptions
235 let dirs
236 try {
237 dirs = await fs.readdir(npxCache, { encoding: 'utf-8' })
238 } catch {
239 output.standard('npx cache does not exist')
240 return
241 }
242 const cache = {}
243 const { default: pMap } = await import('p-map')
244 await pMap(dirs, async e => {
245 const pkgPath = join(npxCache, e)
246 cache[e] = {
247 hash: e,
248 path: pkgPath,
249 valid: false,
250 }
251 try {
252 const pkgJson = await PkgJson.load(pkgPath)
253 cache[e].package = pkgJson.content
254 cache[e].valid = true
255 } catch {
256 // Defaults to not valid already
257 }
258 }, { concurrency: 20 })
259 if (!keys.length) {
260 return cache
261 }
262 const result = {}
263 const abbrevs = abbrev(Object.keys(cache))
264 for (const key of keys) {
265 if (!abbrevs[key]) {
266 throw this.usageError(`Invalid npx key ${key}`)
267 }
268 result[abbrevs[key]] = cache[abbrevs[key]]
269 }
270 return result
271 }
272
273 async npxLs () {
274 const cache = await this.#npxCache()
275 for (const key in cache) {
276 const { hash, valid, package: pkg } = cache[key]
277 let result = `${hash}:`
278 if (!valid) {
279 result = `${result} (empty/invalid)`
280 } else if (pkg?._npx) {
281 result = `${result} ${pkg._npx.packages.join(', ')}`
282 } else {
283 result = `${result} (unknown)`
284 }
285 output.standard(result)
286 }
287 }
288
289 async npxRm (keys) {
290 if (!keys.length) {
291 if (!this.npm.config.get('force')) {
292 throw this.usageError('Please use --force to remove entire npx cache')
293 }
294 const { npxCache } = this.npm.flatOptions
295 if (!this.npm.config.get('dry-run')) {
296 return fs.rm(npxCache, { recursive: true, force: true })
297 }
298 }
299
300 const cache = await this.#npxCache(keys)
301 for (const key in cache) {
302 const { path: cachePath } = cache[key]
303 output.standard(`Removing npx key at ${cachePath}`)
304 if (!this.npm.config.get('dry-run')) {
305 await fs.rm(cachePath, { recursive: true })
306 }
307 }
308 }
309
310 async npxInfo (keys) {
311 const chalk = this.npm.chalk
312 if (!keys.length) {
313 throw this.usageError()
314 }
315 const cache = await this.#npxCache(keys)
316 const Arborist = require('@npmcli/arborist')
317 for (const key in cache) {
318 const { hash, path, package: pkg } = cache[key]
319 let valid = cache[key].valid
320 const results = []
321 try {
322 if (valid) {
323 const arb = new Arborist({ path })
324 const tree = await arb.loadVirtual()
325 if (pkg._npx) {
326 results.push('packages:')
327 for (const p of pkg._npx.packages) {
328 const parsed = npa(p)
329 if (parsed.type === 'directory') {
330 // in the tree the spec is relative, even if the dependency spec is absolute, so we can't find it by name or spec.
331 results.push(`- ${chalk.cyan(p)}`)
332 } else {
333 results.push(`- ${chalk.cyan(p)} (${chalk.blue(tree.children.get(parsed.name).pkgid)})`)
334 }
335 }
336 } else {
337 results.push('packages: (unknown)')
338 results.push(`dependencies:`)
339 for (const dep in pkg.dependencies) {
340 const child = tree.children.get(dep)
341 if (child.isLink) {
342 results.push(`- ${chalk.cyan(child.realpath)}`)
343 } else {
344 results.push(`- ${chalk.cyan(child.pkgid)}`)
345 }
346 }
347 }
348 }
349 } catch (ex) {
350 valid = false
351 }
352 const v = valid ? chalk.green('valid') : chalk.red('invalid')
353 output.standard(`${v} npx cache entry with key ${chalk.blue(hash)}`)
354 output.standard(`location: ${chalk.blue(path)}`)
355 if (valid) {
356 output.standard(results.join('\n'))
357 }
358 output.standard()
359 }
360 }
361}
362
363module.exports = Cache
364