codekingpro/portable-devtools
115k
1const valid = require('semver/functions/valid')
2const clean = require('semver/functions/clean')
3const fs = require('node:fs/promises')
4const path = require('node:path')
5const { log } = require('proc-log')
6const moduleBuiltin = require('node:module')
7
8/**
9 * @type {import('hosted-git-info')}
10 */
11let _hostedGitInfo
12function lazyHostedGitInfo () {
13 if (!_hostedGitInfo) {
14 _hostedGitInfo = require('hosted-git-info')
15 }
16 return _hostedGitInfo
17}
18
19/**
20 * @type {import('glob').glob}
21 */
22let _glob
23function lazyLoadGlob () {
24 if (!_glob) {
25 _glob = require('glob').glob
26 }
27 return _glob
28}
29
30// used to be npm-normalize-package-bin
31function normalizePackageBin (pkg, changes) {
32 if (pkg.bin) {
33 if (typeof pkg.bin === 'string' && pkg.name) {
34 changes?.push('"bin" was converted to an object')
35 pkg.bin = { [pkg.name]: pkg.bin }
36 } else if (Array.isArray(pkg.bin)) {
37 changes?.push('"bin" was converted to an object')
38 pkg.bin = pkg.bin.reduce((acc, k) => {
39 acc[path.basename(k)] = k
40 return acc
41 }, {})
42 }
43 if (typeof pkg.bin === 'object') {
44 for (const binKey in pkg.bin) {
45 if (typeof pkg.bin[binKey] !== 'string') {
46 delete pkg.bin[binKey]
47 changes?.push(`removed invalid "bin[${binKey}]"`)
48 continue
49 }
50 const base = path.basename(secureAndUnixifyPath(binKey))
51 if (!base) {
52 delete pkg.bin[binKey]
53 changes?.push(`removed invalid "bin[${binKey}]"`)
54 continue
55 }
56
57 const binTarget = secureAndUnixifyPath(pkg.bin[binKey])
58
59 if (!binTarget) {
60 delete pkg.bin[binKey]
61 changes?.push(`removed invalid "bin[${binKey}]"`)
62 continue
63 }
64
65 if (base !== binKey) {
66 delete pkg.bin[binKey]
67 changes?.push(`"bin[${binKey}]" was renamed to "bin[${base}]"`)
68 }
69 if (binTarget !== pkg.bin[binKey]) {
70 changes?.push(`"bin[${base}]" script name ${binTarget} was invalid and removed`)
71 }
72 pkg.bin[base] = binTarget
73 }
74
75 if (Object.keys(pkg.bin).length === 0) {
76 changes?.push('empty "bin" was removed')
77 delete pkg.bin
78 }
79
80 return pkg
81 }
82 }
83 delete pkg.bin
84}
85
86function normalizePackageMan (pkg, changes) {
87 if (pkg.man) {
88 const mans = []
89 for (const man of (Array.isArray(pkg.man) ? pkg.man : [pkg.man])) {
90 if (typeof man !== 'string') {
91 changes?.push(`removed invalid "man [${man}]"`)
92 } else {
93 mans.push(secureAndUnixifyPath(man))
94 }
95 }
96
97 if (!mans.length) {
98 changes?.push('empty "man" was removed')
99 } else {
100 pkg.man = mans
101 return pkg
102 }
103 }
104 delete pkg.man
105}
106
107function isCorrectlyEncodedName (spec) {
108 return !spec.match(/[/@\s+%:]/) &&
109 spec === encodeURIComponent(spec)
110}
111
112function isValidScopedPackageName (spec) {
113 if (spec.charAt(0) !== '@') {
114 return false
115 }
116
117 const rest = spec.slice(1).split('/')
118 if (rest.length !== 2) {
119 return false
120 }
121
122 return rest[0] && rest[1] &&
123 rest[0] === encodeURIComponent(rest[0]) &&
124 rest[1] === encodeURIComponent(rest[1])
125}
126
127function unixifyPath (ref) {
128 return ref.replace(/\\|:/g, '/')
129}
130
131function secureAndUnixifyPath (ref) {
132 const secured = unixifyPath(path.join('.', path.join('/', unixifyPath(ref))))
133 return secured.startsWith('./') ? '' : secured
134}
135
136// Only steps that can be ran synchronously. There are some object constructors (i.e. Aborist Node) that need synchronous normalization so here we are.
137function syncSteps (pkg, { strict, steps, changes, allowLegacyCase }) {
138 const data = pkg.content
139 const pkgId = `${data.name ?? ''}@${data.version ?? ''}`
140
141 // name and version are load bearing so we have to clean them up first
142 if (steps.includes('fixName') || steps.includes('fixNameField') || steps.includes('normalizeData')) {
143 if (!data.name && !strict) {
144 changes?.push('Missing "name" field was set to an empty string')
145 data.name = ''
146 } else {
147 if (typeof data.name !== 'string') {
148 throw new Error('name field must be a string.')
149 }
150 if (!strict) {
151 const name = data.name.trim()
152 if (data.name !== name) {
153 changes?.push(`Whitespace was trimmed from "name"`)
154 data.name = name
155 }
156 }
157
158 if (data.name.startsWith('.') ||
159 !(isValidScopedPackageName(data.name) || isCorrectlyEncodedName(data.name)) ||
160 (strict && (!allowLegacyCase) && data.name !== data.name.toLowerCase()) ||
161 data.name.toLowerCase() === 'node_modules' ||
162 data.name.toLowerCase() === 'favicon.ico') {
163 throw new Error('Invalid name: ' + JSON.stringify(data.name))
164 }
165 }
166 }
167
168 if (steps.includes('fixName')) {
169 // Check for conflicts with builtin modules
170 if (moduleBuiltin.builtinModules.includes(data.name)) {
171 log.warn('package-json', pkgId, `Package name "${data.name}" conflicts with a Node.js built-in module name`)
172 }
173 }
174
175 if (steps.includes('fixVersionField') || steps.includes('normalizeData')) {
176 // allow "loose" semver 1.0 versions in non-strict mode
177 // enforce strict semver 2.0 compliance in strict mode
178 const loose = !strict
179 if (!data.version) {
180 data.version = ''
181 } else {
182 if (!valid(data.version, loose)) {
183 throw new Error(`Invalid version: "${data.version}"`)
184 }
185 const version = clean(data.version, loose)
186 if (version !== data.version) {
187 changes?.push(`"version" was cleaned and set to "${version}"`)
188 data.version = version
189 }
190 }
191 }
192
193 // remove attributes that start with "_"
194 if (steps.includes('_attributes')) {
195 for (const key in data) {
196 if (key.startsWith('_')) {
197 changes?.push(`"${key}" was removed`)
198 delete pkg.content[key]
199 }
200 }
201 }
202
203 // build the "_id" attribute
204 if (steps.includes('_id')) {
205 if (data.name && data.version) {
206 changes?.push(`"_id" was set to ${pkgId}`)
207 data._id = pkgId
208 }
209 }
210
211 // fix bundledDependencies typo
212 if (steps.includes('bundledDependencies')) {
213 if (data.bundleDependencies === undefined && data.bundledDependencies !== undefined) {
214 data.bundleDependencies = data.bundledDependencies
215 changes?.push(`Deleted incorrect "bundledDependencies"`)
216 }
217 delete data.bundledDependencies
218 }
219
220 // expand "bundleDependencies: true or translate from object"
221 if (steps.includes('bundleDependencies')) {
222 const bd = data.bundleDependencies
223 if (bd === false && !steps.includes('bundleDependenciesDeleteFalse')) {
224 changes?.push(`"bundleDependencies" was changed from "false" to "[]"`)
225 data.bundleDependencies = []
226 } else if (bd === true) {
227 changes?.push(`"bundleDependencies" was auto-populated from "dependencies"`)
228 data.bundleDependencies = Object.keys(data.dependencies || {})
229 } else if (bd && typeof bd === 'object') {
230 if (!Array.isArray(bd)) {
231 changes?.push(`"bundleDependencies" was changed from an object to an array`)
232 data.bundleDependencies = Object.keys(bd)
233 }
234 } else if ('bundleDependencies' in data) {
235 changes?.push(`"bundleDependencies" was removed`)
236 delete data.bundleDependencies
237 }
238 }
239
240 // it was once common practice to list deps both in optionalDependencies and
241 // in dependencies, to support npm versions that did not know about
242 // optionalDependencies. This is no longer a relevant need, so duplicating
243 // the deps in two places is unnecessary and excessive.
244 if (steps.includes('optionalDedupe')) {
245 if (data.dependencies &&
246 data.optionalDependencies && typeof data.optionalDependencies === 'object') {
247 for (const name in data.optionalDependencies) {
248 changes?.push(`optionalDependencies."${name}" was removed`)
249 delete data.dependencies[name]
250 }
251 if (!Object.keys(data.dependencies).length) {
252 changes?.push(`Empty "optionalDependencies" was removed`)
253 delete data.dependencies
254 }
255 }
256 }
257
258 // strip "node_modules/.bin" from scripts entries
259 // remove invalid scripts entries (non-strings)
260 if ((steps.includes('scripts') || steps.includes('scriptpath')) && data.scripts !== undefined) {
261 const spre = /^(\.[/\\])?node_modules[/\\].bin[\\/]/
262 if (typeof data.scripts === 'object') {
263 for (const name in data.scripts) {
264 if (typeof data.scripts[name] !== 'string') {
265 delete data.scripts[name]
266 changes?.push(`Invalid scripts."${name}" was removed`)
267 } else if (steps.includes('scriptpath') && spre.test(data.scripts[name])) {
268 data.scripts[name] = data.scripts[name].replace(spre, '')
269 changes?.push(`scripts entry "${name}" was fixed to remove node_modules/.bin reference`)
270 }
271 }
272 } else {
273 changes?.push(`Removed invalid "scripts"`)
274 delete data.scripts
275 }
276 }
277
278 if (steps.includes('funding')) {
279 if (data.funding && typeof data.funding === 'string') {
280 data.funding = { url: data.funding }
281 changes?.push(`"funding" was changed to an object with a url attribute`)
282 }
283 }
284
285 // "normalizeData" from "read-package-json", which was just a call through to
286 // "normalize-package-data". We only call the "fixer" functions because
287 // outside of that it was also clobbering _id (which we already conditionally
288 // do) and also adding the gypfile script (which we also already
289 // conditionally do)
290
291 // Some steps are isolated so we can do a limited subset of these in `fix`
292 if (steps.includes('fixRepositoryField') || steps.includes('normalizeData')) {
293 if (data.repositories) {
294 changes?.push(`"repository" was set to the first entry in "repositories" (${data.repository})`)
295 data.repository = data.repositories[0]
296 }
297 if (data.repository) {
298 if (typeof data.repository === 'string') {
299 changes?.push('"repository" was changed from a string to an object')
300 data.repository = {
301 type: 'git',
302 url: data.repository,
303 }
304 }
305 if (data.repository.url) {
306 const hosted = lazyHostedGitInfo().fromUrl(data.repository.url)
307 let r
308 if (hosted) {
309 if (hosted.getDefaultRepresentation() === 'shortcut') {
310 r = hosted.https()
311 } else {
312 r = hosted.toString()
313 }
314 if (r !== data.repository.url) {
315 changes?.push(`"repository.url" was normalized to "${r}"`)
316 data.repository.url = r
317 }
318 }
319 }
320 }
321 }
322
323 if (steps.includes('fixDependencies') || steps.includes('normalizeData')) {
324 // peerDependencies?
325 // devDependencies is meaningless here, it's ignored on an installed package
326 for (const type of ['dependencies', 'devDependencies', 'optionalDependencies']) {
327 if (data[type]) {
328 let secondWarning = true
329 if (typeof data[type] === 'string') {
330 changes?.push(`"${type}" was converted from a string into an object`)
331 data[type] = data[type].trim().split(/[\n\r\s\t ,]+/)
332 secondWarning = false
333 }
334 if (Array.isArray(data[type])) {
335 if (secondWarning) {
336 changes?.push(`"${type}" was converted from an array into an object`)
337 }
338 const o = {}
339 for (const d of data[type]) {
340 if (typeof d === 'string') {
341 const dep = d.trim().split(/(:?[@\s><=])/)
342 const dn = dep.shift()
343 const dv = dep.join('').replace(/^@/, '').trim()
344 o[dn] = dv
345 }
346 }
347 data[type] = o
348 }
349 }
350 }
351 // normalize-package-data used to put optional dependencies BACK into
352 // dependencies here, we no longer do this
353
354 for (const deps of ['dependencies', 'devDependencies']) {
355 if (deps in data) {
356 if (!data[deps] || typeof data[deps] !== 'object') {
357 changes?.push(`Removed invalid "${deps}"`)
358 delete data[deps]
359 } else {
360 for (const d in data[deps]) {
361 const r = data[deps][d]
362 if (typeof r !== 'string') {
363 changes?.push(`Removed invalid "${deps}.${d}"`)
364 delete data[deps][d]
365 }
366 const hosted = lazyHostedGitInfo().fromUrl(data[deps][d])?.toString()
367 if (hosted && hosted !== data[deps][d]) {
368 changes?.push(`Normalized git reference to "${deps}.${d}"`)
369 data[deps][d] = hosted.toString()
370 }
371 }
372 }
373 }
374 }
375 }
376
377 // TODO some of this is duplicated in other steps here, a future breaking change may be able to remove the duplicates involved in this step
378 if (steps.includes('normalizeData')) {
379 const { normalizeData } = require('./normalize-data.js')
380 normalizeData(data, changes)
381 }
382}
383
384// Steps that require await, distinct from sync-steps.js
385async function asyncSteps (pkg, { steps, root, changes }) {
386 const data = pkg.content
387 const scripts = data.scripts || {}
388 const pkgId = `${data.name ?? ''}@${data.version ?? ''}`
389
390 // add "install" attribute if any "*.gyp" files exist
391 if (steps.includes('gypfile')) {
392 if (!scripts.install && !scripts.preinstall && data.gypfile !== false) {
393 const files = await lazyLoadGlob()('*.gyp', { cwd: pkg.path })
394 if (files.length) {
395 scripts.install = 'node-gyp rebuild'
396 data.scripts = scripts
397 data.gypfile = true
398 changes?.push(`"scripts.install" was set to "node-gyp rebuild"`)
399 changes?.push(`"gypfile" was set to "true"`)
400 }
401 }
402 }
403
404 // add "start" attribute if "server.js" exists
405 if (steps.includes('serverjs') && !scripts.start) {
406 try {
407 await fs.access(path.join(pkg.path, 'server.js'))
408 scripts.start = 'node server.js'
409 data.scripts = scripts
410 changes?.push('"scripts.start" was set to "node server.js"')
411 } catch {
412 // do nothing
413 }
414 }
415
416 // populate "authors" attribute
417 if (steps.includes('authors') && !data.contributors) {
418 try {
419 const authorData = await fs.readFile(path.join(pkg.path, 'AUTHORS'), 'utf8')
420 const authors = authorData.split(/\r?\n/g)
421 .map(line => line.replace(/^\s*#.*$/, '').trim())
422 .filter(line => line)
423 data.contributors = authors
424 changes?.push('"contributors" was auto-populated with the contents of the "AUTHORS" file')
425 } catch {
426 // do nothing
427 }
428 }
429
430 // populate "readme" attribute
431 if (steps.includes('readme') && !data.readme) {
432 const mdre = /\.m?a?r?k?d?o?w?n?$/i
433 const files = await lazyLoadGlob()('{README,README.*}', {
434 cwd: pkg.path,
435 nocase: true,
436 mark: true,
437 })
438 let readmeFile
439 for (const file of files) {
440 // don't accept directories.
441 if (!file.endsWith(path.sep)) {
442 if (file.match(mdre)) {
443 readmeFile = file
444 break
445 }
446 if (file.endsWith('README')) {
447 readmeFile = file
448 }
449 }
450 }
451 if (readmeFile) {
452 const readmeData = await fs.readFile(path.join(pkg.path, readmeFile), 'utf8')
453 data.readme = readmeData
454 data.readmeFilename = readmeFile
455 changes?.push(`"readme" was set to the contents of ${readmeFile}`)
456 changes?.push(`"readmeFilename" was set to ${readmeFile}`)
457 }
458 if (!data.readme) {
459 data.readme = 'ERROR: No README data found!'
460 }
461 }
462
463 // expand directories.man
464 if (steps.includes('mans')) {
465 if (data.directories?.man && !data.man) {
466 const manDir = secureAndUnixifyPath(data.directories.man)
467 const cwd = path.resolve(pkg.path, manDir)
468 const files = await lazyLoadGlob()('**/*.[0-9]', { cwd })
469 data.man = files.map(man =>
470 path.relative(pkg.path, path.join(cwd, man)).split(path.sep).join('/')
471 )
472 }
473 normalizePackageMan(data, changes)
474 }
475
476 // expand "directories.bin"
477 if (steps.includes('binDir') && data.directories?.bin && !data.bin && pkg.path) {
478 const binPath = secureAndUnixifyPath(data.directories.bin)
479 const bins = await lazyLoadGlob()('**', { cwd: path.resolve(pkg.path, binPath) })
480 data.bin = bins.reduce((acc, binFile) => {
481 if (binFile && !binFile.startsWith('.')) {
482 const binName = path.basename(binFile)
483 // binPath is already cleaned and unixified, no need to path.join here.
484 acc[binName] = `${binPath}/${secureAndUnixifyPath(binFile)}`
485 }
486 return acc
487 }, {})
488 } else if (steps.includes('bin') || steps.includes('binDir') || steps.includes('binRefs')) {
489 normalizePackageBin(data, changes)
490 }
491
492 // populate "gitHead" attribute
493 if (steps.includes('gitHead') && !data.gitHead) {
494 const git = require('@npmcli/git')
495 const gitRoot = await git.find({ cwd: pkg.path, root })
496 let head
497 if (gitRoot) {
498 try {
499 head = await fs.readFile(path.resolve(gitRoot, '.git/HEAD'), 'utf8')
500 } catch (err) {
501 // do nothing
502 }
503 }
504 let headData
505 if (head) {
506 if (head.startsWith('ref: ')) {
507 const headRef = head.replace(/^ref: /, '').trim()
508 const headFile = path.resolve(gitRoot, '.git', headRef)
509 try {
510 headData = await fs.readFile(headFile, 'utf8')
511 headData = headData.replace(/^ref: /, '').trim()
512 } catch (err) {
513 // do nothing
514 }
515 if (!headData) {
516 const packFile = path.resolve(gitRoot, '.git/packed-refs')
517 try {
518 let refs = await fs.readFile(packFile, 'utf8')
519 if (refs) {
520 refs = refs.split('\n')
521 for (let i = 0; i < refs.length; i++) {
522 const match = refs[i].match(/^([0-9a-f]{40}) (.+)$/)
523 if (match && match[2].trim() === headRef) {
524 headData = match[1]
525 break
526 }
527 }
528 }
529 } catch {
530 // do nothing
531 }
532 }
533 } else {
534 headData = head.trim()
535 }
536 }
537 if (headData) {
538 data.gitHead = headData
539 }
540 }
541
542 // populate "types" attribute
543 if (steps.includes('fillTypes')) {
544 const index = data.main || 'index.js'
545
546 if (typeof index !== 'string') {
547 throw new TypeError('The "main" attribute must be of type string.')
548 }
549
550 // TODO exports is much more complicated than this in verbose format
551 // We need to support for instance
552
553 // "exports": {
554 // ".": [
555 // {
556 // "default": "./lib/npm.js"
557 // },
558 // "./lib/npm.js"
559 // ],
560 // "./package.json": "./package.json"
561 // },
562 // as well as conditional exports
563
564 // if (data.exports && typeof data.exports === 'string') {
565 // index = data.exports
566 // }
567
568 // if (data.exports && data.exports['.']) {
569 // index = data.exports['.']
570 // if (typeof index !== 'string') {
571 // }
572 // }
573 const extless = path.join(path.dirname(index), path.basename(index, path.extname(index)))
574 const dts = `./${extless}.d.ts`
575 const hasDTSFields = 'types' in data || 'typings' in data
576 if (!hasDTSFields) {
577 try {
578 await fs.access(path.join(pkg.path, dts))
579 data.types = dts.split(path.sep).join('/')
580 } catch {
581 // do nothing
582 }
583 }
584 }
585
586 // Warn if the bin references don't point to anything. This might be better
587 // in normalize-package-data if it had access to the file path.
588 if (steps.includes('binRefs') && data.bin instanceof Object) {
589 for (const key in data.bin) {
590 try {
591 await fs.access(path.resolve(pkg.path, data.bin[key]))
592 } catch {
593 log.warn('package-json', pkgId, `No bin file found at ${data.bin[key]}`)
594 // XXX: should a future breaking change delete bin entries that cannot be accessed?
595 }
596 }
597 }
598}
599
600// We don't want the `changes` array in here by default because this is a hot path for parsing packuments during install. The calling method passes it in if it wants to track changes.
601async function normalize (pkg, opts) {
602 if (!pkg.content) {
603 throw new Error('Can not normalize without content')
604 }
605 await asyncSteps(pkg, opts)
606 // the normalizeData part of this needs to be the last thing ran, so sync comes second
607 syncSteps(pkg, opts)
608}
609
610function syncNormalize (pkg, opts) {
611 syncSteps(pkg, opts)
612}
613
614module.exports = { normalize, syncNormalize }
615