codekingpro/portable-devtools
114k
1const cacache = require('cacache')
2const git = require('@npmcli/git')
3const npa = require('npm-package-arg')
4const pickManifest = require('npm-pick-manifest')
5const { Minipass } = require('minipass')
6const { log } = require('proc-log')
7const DirFetcher = require('./dir.js')
8const Fetcher = require('./fetcher.js')
9const FileFetcher = require('./file.js')
10const RemoteFetcher = require('./remote.js')
11const _ = require('./util/protected.js')
12const addGitSha = require('./util/add-git-sha.js')
13const npm = require('./util/npm.js')
14
15const hashre = /^[a-f0-9]{40,64}$/
16
17// get the repository url.
18// prefer https if there's auth, since ssh will drop that.
19// otherwise, prefer ssh if available (more secure).
20// We have to add the git+ back because npa suppresses it.
21const repoUrl = (h, opts) =>
22 h.sshurl && !(h.https && h.auth) && addGitPlus(h.sshurl(opts)) ||
23 h.https && addGitPlus(h.https(opts))
24
25// add git+ to the url, but only one time.
26const addGitPlus = url => url && `git+${url}`.replace(/^(git\+)+/, 'git+')
27
28const checkoutError = (expected, found) => {
29 const err = new Error(`Commit mismatch: expected SHA ${expected} and cloned HEAD ${found}`)
30 err.code = 'EGITCHECKOUT'
31 err.sha = expected
32 err.head = found
33 return err
34}
35
36class GitFetcher extends Fetcher {
37 constructor (spec, opts) {
38 super(spec, opts)
39
40 // we never want to compare integrity for git dependencies: npm/rfcs#525
41 if (this.opts.integrity) {
42 delete this.opts.integrity
43 log.warn(`skipping integrity check for git dependency ${this.spec.fetchSpec}`)
44 }
45
46 this.resolvedRef = null
47 if (this.spec.hosted) {
48 this.from = this.spec.hosted.shortcut({ noCommittish: false })
49 }
50
51 // shortcut: avoid full clone when we can go straight to the tgz
52 // if we have the full sha and it's a hosted git platform
53 if (this.spec.gitCommittish && hashre.test(this.spec.gitCommittish)) {
54 this.resolvedSha = this.spec.gitCommittish
55 // use hosted.tarball() when we shell to RemoteFetcher later
56 this.resolved = this.spec.hosted
57 ? repoUrl(this.spec.hosted, { noCommittish: false })
58 : this.spec.rawSpec
59 } else {
60 this.resolvedSha = ''
61 }
62
63 this.Arborist = opts.Arborist || null
64 }
65
66 // just exposed to make it easier to test all the combinations
67 static repoUrl (hosted, opts) {
68 return repoUrl(hosted, opts)
69 }
70
71 get types () {
72 return ['git']
73 }
74
75 resolve () {
76 // likely a hosted git repo with a sha, so get the tarball url
77 // but in general, no reason to resolve() more than necessary!
78 if (this.resolved) {
79 return super.resolve()
80 }
81
82 // fetch the git repo and then look at the current hash
83 const h = this.spec.hosted
84 // try to use ssh, fall back to git.
85 return h
86 ? this.#resolvedFromHosted(h)
87 : this.#resolvedFromRepo(this.spec.fetchSpec)
88 }
89
90 // first try https, since that's faster and passphrase-less for
91 // public repos, and supports private repos when auth is provided.
92 // Fall back to SSH to support private repos
93 // NB: we always store the https url in resolved field if auth
94 // is present, otherwise ssh if the hosted type provides it
95 #resolvedFromHosted (hosted) {
96 return this.#resolvedFromRepo(hosted.https && hosted.https()).catch(er => {
97 // Throw early since we know pathspec errors will fail again if retried
98 if (er instanceof git.errors.GitPathspecError) {
99 throw er
100 }
101 const ssh = hosted.sshurl && hosted.sshurl()
102 // no fallthrough if we can't fall through or have https auth
103 if (!ssh || hosted.auth) {
104 throw er
105 }
106 return this.#resolvedFromRepo(ssh)
107 })
108 }
109
110 #resolvedFromRepo (gitRemote) {
111 // XXX make this a custom error class
112 if (!gitRemote) {
113 return Promise.reject(new Error(`No git url for ${this.spec}`))
114 }
115 const gitRange = this.spec.gitRange
116 const name = this.spec.name
117 return git.revs(gitRemote, this.opts).then(remoteRefs => {
118 return gitRange ? pickManifest({
119 versions: remoteRefs.versions,
120 'dist-tags': remoteRefs['dist-tags'],
121 name,
122 }, gitRange, this.opts)
123 : this.spec.gitCommittish ?
124 remoteRefs.refs[this.spec.gitCommittish] ||
125 remoteRefs.refs[remoteRefs.shas[this.spec.gitCommittish]]
126 : remoteRefs.refs.HEAD // no git committish, get default head
127 }).then(revDoc => {
128 // the committish provided isn't in the rev list
129 // things like HEAD~3 or @yesterday can land here.
130 if (!revDoc || !revDoc.sha) {
131 return this.#resolvedFromClone()
132 }
133
134 this.resolvedRef = revDoc
135 this.resolvedSha = revDoc.sha
136 this.#addGitSha(revDoc.sha)
137 return this.resolved
138 })
139 }
140
141 #setResolvedWithSha (withSha) {
142 // we haven't cloned, so a tgz download is still faster
143 // of course, if it's not a known host, we can't do that.
144 this.resolved = !this.spec.hosted ? withSha
145 : repoUrl(npa(withSha).hosted, { noCommittish: false })
146 }
147
148 // when we get the git sha, we affix it to our spec to build up
149 // either a git url with a hash, or a tarball download URL
150 #addGitSha (sha) {
151 this.#setResolvedWithSha(addGitSha(this.spec, sha))
152 }
153
154 #resolvedFromClone () {
155 // do a full or shallow clone, then look at the HEAD
156 // kind of wasteful, but no other option, really
157 return this.#clone(() => this.resolved)
158 }
159
160 #prepareDir (dir) {
161 return this[_.readPackageJson](dir).then(mani => {
162 // no need if we aren't going to do any preparation.
163 const scripts = mani.scripts
164 if (!mani.workspaces && (!scripts || !(
165 scripts.postinstall ||
166 scripts.build ||
167 scripts.preinstall ||
168 scripts.install ||
169 scripts.prepack ||
170 scripts.prepare))) {
171 return
172 }
173
174 // to avoid cases where we have an cycle of git deps that depend
175 // on one another, we only ever do preparation for one instance
176 // of a given git dep along the chain of installations.
177 // Note that this does mean that a dependency MAY in theory end up
178 // trying to run its prepare script using a dependency that has not
179 // been properly prepared itself, but that edge case is smaller
180 // and less hazardous than a fork bomb of npm and git commands.
181 const noPrepare = !process.env._PACOTE_NO_PREPARE_ ? []
182 : process.env._PACOTE_NO_PREPARE_.split('\n')
183 if (noPrepare.includes(this.resolved)) {
184 log.info('prepare', 'skip prepare, already seen', this.resolved)
185 return
186 }
187 noPrepare.push(this.resolved)
188
189 // the DirFetcher will do its own preparation to run the prepare scripts
190 // All we have to do is put the deps in place so that it can succeed.
191 return npm(
192 this.npmBin,
193 [].concat(this.npmInstallCmd).concat(this.npmCliConfig),
194 dir,
195 { ...process.env, _PACOTE_NO_PREPARE_: noPrepare.join('\n') },
196 { message: 'git dep preparation failed' }
197 )
198 })
199 }
200
201 [_.tarballFromResolved] () {
202 const stream = new Minipass()
203 stream.resolved = this.resolved
204 stream.from = this.from
205
206 // check it out and then shell out to the DirFetcher tarball packer
207 this.#clone(dir => this.#prepareDir(dir)
208 .then(() => new Promise((res, rej) => {
209 if (!this.Arborist) {
210 throw new Error('GitFetcher requires an Arborist constructor to pack a tarball')
211 }
212 const df = new DirFetcher(`file:${dir}`, {
213 ...this.opts,
214 Arborist: this.Arborist,
215 resolved: null,
216 integrity: null,
217 })
218 const dirStream = df[_.tarballFromResolved]()
219 dirStream.on('error', rej)
220 dirStream.on('end', res)
221 dirStream.pipe(stream)
222 }))).catch(
223 /* istanbul ignore next: very unlikely and hard to test */
224 er => stream.emit('error', er)
225 )
226 return stream
227 }
228
229 // clone a git repo into a temp folder (or fetch and unpack if possible)
230 // handler accepts a directory, and returns a promise that resolves
231 // when we're done with it, at which point, cacache deletes it
232 //
233 // TODO: after cloning, create a tarball of the folder, and add to the cache
234 // with cacache.put.stream(), using a key that's deterministic based on the
235 // spec and repo, so that we don't ever clone the same thing multiple times.
236 #clone (handler, tarballOk = true) {
237 const o = { tmpPrefix: 'git-clone' }
238 const ref = this.resolvedSha || this.spec.gitCommittish
239 const h = this.spec.hosted
240 const resolved = this.resolved
241
242 // can be set manually to false to fall back to actual git clone
243 tarballOk = tarballOk &&
244 h && resolved === repoUrl(h, { noCommittish: false }) && h.tarball
245
246 return cacache.tmp.withTmp(this.cache, o, async tmp => {
247 // if we're resolved, and have a tarball url, shell out to RemoteFetcher
248 if (tarballOk) {
249 const nameat = this.spec.name ? `${this.spec.name}@` : ''
250 return new RemoteFetcher(h.tarball({ noCommittish: false }), {
251 ...this.opts,
252 allowGitIgnore: true,
253 pkgid: `git:${nameat}${this.resolved}`,
254 resolved: this.resolved,
255 integrity: null, // it'll always be different, if we have one
256 }).extract(tmp).then(() => handler(`${tmp}${this.spec.gitSubdir || ''}`), er => {
257 // fall back to ssh download if tarball fails
258 if (er.constructor.name.match(/^Http/)) {
259 return this.#clone(handler, false)
260 } else {
261 throw er
262 }
263 })
264 }
265
266 const sha = await (
267 h ? this.#cloneHosted(ref, tmp)
268 : this.#cloneRepo(this.spec.fetchSpec, ref, tmp)
269 )
270 // if we already have a resolved sha ensure it doesn't change
271 if (this.resolvedSha && this.resolvedSha !== sha) {
272 throw checkoutError(this.resolvedSha, sha)
273 }
274 this.resolvedSha = sha
275 if (!this.resolved) {
276 await this.#addGitSha(sha)
277 }
278 return handler(`${tmp}${this.spec.gitSubdir || ''}`)
279 })
280 }
281
282 // first try https, since that's faster and passphrase-less for
283 // public repos, and supports private repos when auth is provided.
284 // Fall back to SSH to support private repos
285 // NB: we always store the https url in resolved field if auth
286 // is present, otherwise ssh if the hosted type provides it
287 #cloneHosted (ref, tmp) {
288 const hosted = this.spec.hosted
289 return this.#cloneRepo(hosted.https({ noCommittish: true }), ref, tmp)
290 .catch(er => {
291 // Throw early since we know pathspec errors will fail again if retried
292 if (er instanceof git.errors.GitPathspecError) {
293 throw er
294 }
295 const ssh = hosted.sshurl && hosted.sshurl({ noCommittish: true })
296 // no fallthrough if we can't fall through or have https auth
297 if (!ssh || hosted.auth) {
298 throw er
299 }
300 return this.#cloneRepo(ssh, ref, tmp)
301 })
302 }
303
304 #cloneRepo (repo, ref, tmp) {
305 const { opts, spec } = this
306 return git.clone(repo, ref, tmp, { ...opts, spec })
307 }
308
309 manifest () {
310 if (this.package) {
311 return Promise.resolve(this.package)
312 }
313
314 return this.spec.hosted && this.resolved
315 ? FileFetcher.prototype.manifest.apply(this)
316 : this.#clone(dir =>
317 this[_.readPackageJson](dir)
318 .then(mani => this.package = {
319 ...mani,
320 _resolved: this.resolved,
321 _from: this.from,
322 }))
323 }
324
325 packument () {
326 return FileFetcher.prototype.packument.apply(this)
327 }
328}
329module.exports = GitFetcher
330