codekingpro/portable-devtools
114k
1'use strict'
2const { URL } = require('url')
3const http = require('http')
4const https = require('https')
5const zlib = require('minizlib')
6const { Minipass } = require('minipass')
7
8const Body = require('./body.js')
9const { writeToStream, getTotalBytes } = Body
10const Response = require('./response.js')
11const Headers = require('./headers.js')
12const { createHeadersLenient } = Headers
13const Request = require('./request.js')
14const { getNodeRequestOptions } = Request
15const FetchError = require('./fetch-error.js')
16const AbortError = require('./abort-error.js')
17
18// XXX this should really be split up and unit-ized for easier testing
19// and better DRY implementation of data/http request aborting
20const fetch = async (url, opts) => {
21 if (/^data:/.test(url)) {
22 const request = new Request(url, opts)
23 // delay 1 promise tick so that the consumer can abort right away
24 return Promise.resolve().then(() => new Promise((resolve, reject) => {
25 let type, data
26 try {
27 const { pathname, search } = new URL(url)
28 const split = pathname.split(',')
29 if (split.length < 2) {
30 throw new Error('invalid data: URI')
31 }
32 const mime = split.shift()
33 const base64 = /;base64$/.test(mime)
34 type = base64 ? mime.slice(0, -1 * ';base64'.length) : mime
35 const rawData = decodeURIComponent(split.join(',') + search)
36 data = base64 ? Buffer.from(rawData, 'base64') : Buffer.from(rawData)
37 } catch (er) {
38 return reject(new FetchError(`[${request.method}] ${
39 request.url} invalid URL, ${er.message}`, 'system', er))
40 }
41
42 const { signal } = request
43 if (signal && signal.aborted) {
44 return reject(new AbortError('The user aborted a request.'))
45 }
46
47 const headers = { 'Content-Length': data.length }
48 if (type) {
49 headers['Content-Type'] = type
50 }
51 return resolve(new Response(data, { headers }))
52 }))
53 }
54
55 return new Promise((resolve, reject) => {
56 // build request object
57 const request = new Request(url, opts)
58 let options
59 try {
60 options = getNodeRequestOptions(request)
61 } catch (er) {
62 return reject(er)
63 }
64
65 const send = (options.protocol === 'https:' ? https : http).request
66 const { signal } = request
67 let response = null
68 const abort = () => {
69 const error = new AbortError('The user aborted a request.')
70 reject(error)
71 if (Minipass.isStream(request.body) &&
72 typeof request.body.destroy === 'function') {
73 request.body.destroy(error)
74 }
75 if (response && response.body) {
76 response.body.emit('error', error)
77 }
78 }
79
80 if (signal && signal.aborted) {
81 return abort()
82 }
83
84 const abortAndFinalize = () => {
85 abort()
86 finalize()
87 }
88
89 const finalize = () => {
90 req.abort()
91 if (signal) {
92 signal.removeEventListener('abort', abortAndFinalize)
93 }
94 clearTimeout(reqTimeout)
95 }
96
97 // send request
98 const req = send(options)
99
100 if (signal) {
101 signal.addEventListener('abort', abortAndFinalize)
102 }
103
104 let reqTimeout = null
105 if (request.timeout) {
106 req.once('socket', () => {
107 reqTimeout = setTimeout(() => {
108 reject(new FetchError(`network timeout at: ${
109 request.url}`, 'request-timeout'))
110 finalize()
111 }, request.timeout)
112 })
113 }
114
115 req.on('error', er => {
116 // if a 'response' event is emitted before the 'error' event, then by the
117 // time this handler is run it's too late to reject the Promise for the
118 // response. instead, we forward the error event to the response stream
119 // so that the error will surface to the user when they try to consume
120 // the body. this is done as a side effect of aborting the request except
121 // for in windows, where we must forward the event manually, otherwise
122 // there is no longer a ref'd socket attached to the request and the
123 // stream never ends so the event loop runs out of work and the process
124 // exits without warning.
125 // coverage skipped here due to the difficulty in testing
126 // istanbul ignore next
127 if (req.res) {
128 req.res.emit('error', er)
129 }
130 reject(new FetchError(`request to ${request.url} failed, reason: ${
131 er.message}`, 'system', er))
132 finalize()
133 })
134
135 req.on('response', res => {
136 clearTimeout(reqTimeout)
137
138 const headers = createHeadersLenient(res.headers)
139
140 // HTTP fetch step 5
141 if (fetch.isRedirect(res.statusCode)) {
142 // HTTP fetch step 5.2
143 const location = headers.get('Location')
144
145 // HTTP fetch step 5.3
146 let locationURL = null
147 try {
148 locationURL = location === null ? null : new URL(location, request.url).toString()
149 } catch {
150 // error here can only be invalid URL in Location: header
151 // do not throw when options.redirect == manual
152 // let the user extract the errorneous redirect URL
153 if (request.redirect !== 'manual') {
154 /* eslint-disable-next-line max-len */
155 reject(new FetchError(`uri requested responds with an invalid redirect URL: ${location}`, 'invalid-redirect'))
156 finalize()
157 return
158 }
159 }
160
161 // HTTP fetch step 5.5
162 if (request.redirect === 'error') {
163 reject(new FetchError('uri requested responds with a redirect, ' +
164 `redirect mode is set to error: ${request.url}`, 'no-redirect'))
165 finalize()
166 return
167 } else if (request.redirect === 'manual') {
168 // node-fetch-specific step: make manual redirect a bit easier to
169 // use by setting the Location header value to the resolved URL.
170 if (locationURL !== null) {
171 // handle corrupted header
172 try {
173 headers.set('Location', locationURL)
174 } catch (err) {
175 /* istanbul ignore next: nodejs server prevent invalid
176 response headers, we can't test this through normal
177 request */
178 reject(err)
179 }
180 }
181 } else if (request.redirect === 'follow' && locationURL !== null) {
182 // HTTP-redirect fetch step 5
183 if (request.counter >= request.follow) {
184 reject(new FetchError(`maximum redirect reached at: ${
185 request.url}`, 'max-redirect'))
186 finalize()
187 return
188 }
189
190 // HTTP-redirect fetch step 9
191 if (res.statusCode !== 303 &&
192 request.body &&
193 getTotalBytes(request) === null) {
194 reject(new FetchError(
195 'Cannot follow redirect with body being a readable stream',
196 'unsupported-redirect'
197 ))
198 finalize()
199 return
200 }
201
202 // Update host due to redirection
203 request.headers.set('host', (new URL(locationURL)).host)
204
205 // HTTP-redirect fetch step 6 (counter increment)
206 // Create a new Request object.
207 const requestOpts = {
208 headers: new Headers(request.headers),
209 follow: request.follow,
210 counter: request.counter + 1,
211 agent: request.agent,
212 compress: request.compress,
213 method: request.method,
214 body: request.body,
215 signal: request.signal,
216 timeout: request.timeout,
217 }
218
219 // if the redirect is to a new hostname, strip the authorization and cookie headers
220 const parsedOriginal = new URL(request.url)
221 const parsedRedirect = new URL(locationURL)
222 if (parsedOriginal.hostname !== parsedRedirect.hostname) {
223 requestOpts.headers.delete('authorization')
224 requestOpts.headers.delete('cookie')
225 }
226
227 // HTTP-redirect fetch step 11
228 if (res.statusCode === 303 || (
229 (res.statusCode === 301 || res.statusCode === 302) &&
230 request.method === 'POST'
231 )) {
232 requestOpts.method = 'GET'
233 requestOpts.body = undefined
234 requestOpts.headers.delete('content-length')
235 }
236
237 // HTTP-redirect fetch step 15
238 resolve(fetch(new Request(locationURL, requestOpts)))
239 finalize()
240 return
241 }
242 } // end if(isRedirect)
243
244 // prepare response
245 res.once('end', () =>
246 signal && signal.removeEventListener('abort', abortAndFinalize))
247
248 const body = new Minipass()
249 // if an error occurs, either on the response stream itself, on one of the
250 // decoder streams, or a response length timeout from the Body class, we
251 // forward the error through to our internal body stream. If we see an
252 // error event on that, we call finalize to abort the request and ensure
253 // we don't leave a socket believing a request is in flight.
254 // this is difficult to test, so lacks specific coverage.
255 body.on('error', finalize)
256 // exceedingly rare that the stream would have an error,
257 // but just in case we proxy it to the stream in use.
258 res.on('error', /* istanbul ignore next */ er => body.emit('error', er))
259 res.on('data', (chunk) => body.write(chunk))
260 res.on('end', () => body.end())
261
262 const responseOptions = {
263 url: request.url,
264 status: res.statusCode,
265 statusText: res.statusMessage,
266 headers: headers,
267 size: request.size,
268 timeout: request.timeout,
269 counter: request.counter,
270 trailer: new Promise(resolveTrailer =>
271 res.on('end', () => resolveTrailer(createHeadersLenient(res.trailers)))),
272 }
273
274 // HTTP-network fetch step 12.1.1.3
275 const codings = headers.get('Content-Encoding')
276
277 // HTTP-network fetch step 12.1.1.4: handle content codings
278
279 // in following scenarios we ignore compression support
280 // 1. compression support is disabled
281 // 2. HEAD request
282 // 3. no Content-Encoding header
283 // 4. no content response (204)
284 // 5. content not modified response (304)
285 if (!request.compress ||
286 request.method === 'HEAD' ||
287 codings === null ||
288 res.statusCode === 204 ||
289 res.statusCode === 304) {
290 response = new Response(body, responseOptions)
291 resolve(response)
292 return
293 }
294
295 // Be less strict when decoding compressed responses, since sometimes
296 // servers send slightly invalid responses that are still accepted
297 // by common browsers.
298 // Always using Z_SYNC_FLUSH is what cURL does.
299 const zlibOptions = {
300 flush: zlib.constants.Z_SYNC_FLUSH,
301 finishFlush: zlib.constants.Z_SYNC_FLUSH,
302 }
303
304 // for gzip
305 if (codings === 'gzip' || codings === 'x-gzip') {
306 const unzip = new zlib.Gunzip(zlibOptions)
307 response = new Response(
308 // exceedingly rare that the stream would have an error,
309 // but just in case we proxy it to the stream in use.
310 body.on('error', /* istanbul ignore next */ er => unzip.emit('error', er)).pipe(unzip),
311 responseOptions
312 )
313 resolve(response)
314 return
315 }
316
317 // for deflate
318 if (codings === 'deflate' || codings === 'x-deflate') {
319 // handle the infamous raw deflate response from old servers
320 // a hack for old IIS and Apache servers
321 res.once('data', chunk => {
322 // see http://stackoverflow.com/questions/37519828
323 const decoder = (chunk[0] & 0x0F) === 0x08
324 ? new zlib.Inflate()
325 : new zlib.InflateRaw()
326 // exceedingly rare that the stream would have an error,
327 // but just in case we proxy it to the stream in use.
328 body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
329 response = new Response(decoder, responseOptions)
330 resolve(response)
331 })
332 return
333 }
334
335 // for br
336 if (codings === 'br') {
337 // ignoring coverage so tests don't have to fake support (or lack of) for brotli
338 // istanbul ignore next
339 try {
340 var decoder = new zlib.BrotliDecompress()
341 } catch (err) {
342 reject(err)
343 finalize()
344 return
345 }
346 // exceedingly rare that the stream would have an error,
347 // but just in case we proxy it to the stream in use.
348 body.on('error', /* istanbul ignore next */ er => decoder.emit('error', er)).pipe(decoder)
349 response = new Response(decoder, responseOptions)
350 resolve(response)
351 return
352 }
353
354 // otherwise, use response as-is
355 response = new Response(body, responseOptions)
356 resolve(response)
357 })
358
359 writeToStream(req, request)
360 })
361}
362
363module.exports = fetch
364
365fetch.isRedirect = code =>
366 code === 301 ||
367 code === 302 ||
368 code === 303 ||
369 code === 307 ||
370 code === 308
371
372fetch.Headers = Headers
373fetch.Request = Request
374fetch.Response = Response
375fetch.FetchError = FetchError
376fetch.AbortError = AbortError
377