codekingpro/portable-devtools
114k
1// https://github.com/Ethan-Arrowood/undici-fetch
2
3'use strict'
4
5const {
6 makeNetworkError,
7 makeAppropriateNetworkError,
8 filterResponse,
9 makeResponse,
10 fromInnerResponse
11} = require('./response')
12const { HeadersList } = require('./headers')
13const { Request, cloneRequest } = require('./request')
14const zlib = require('node:zlib')
15const {
16 bytesMatch,
17 makePolicyContainer,
18 clonePolicyContainer,
19 requestBadPort,
20 TAOCheck,
21 appendRequestOriginHeader,
22 responseLocationURL,
23 requestCurrentURL,
24 setRequestReferrerPolicyOnRedirect,
25 tryUpgradeRequestToAPotentiallyTrustworthyURL,
26 createOpaqueTimingInfo,
27 appendFetchMetadata,
28 corsCheck,
29 crossOriginResourcePolicyCheck,
30 determineRequestsReferrer,
31 coarsenedSharedCurrentTime,
32 createDeferredPromise,
33 isBlobLike,
34 sameOrigin,
35 isCancelled,
36 isAborted,
37 isErrorLike,
38 fullyReadBody,
39 readableStreamClose,
40 isomorphicEncode,
41 urlIsLocal,
42 urlIsHttpHttpsScheme,
43 urlHasHttpsScheme,
44 clampAndCoarsenConnectionTimingInfo,
45 simpleRangeHeaderValue,
46 buildContentRange,
47 createInflate,
48 extractMimeType
49} = require('./util')
50const { kState, kDispatcher } = require('./symbols')
51const assert = require('node:assert')
52const { safelyExtractBody, extractBody } = require('./body')
53const {
54 redirectStatusSet,
55 nullBodyStatus,
56 safeMethodsSet,
57 requestBodyHeader,
58 subresourceSet
59} = require('./constants')
60const EE = require('node:events')
61const { Readable, pipeline, finished } = require('node:stream')
62const { addAbortListener, isErrored, isReadable, bufferToLowerCasedHeaderName } = require('../../core/util')
63const { dataURLProcessor, serializeAMimeType, minimizeSupportedMimeType } = require('./data-url')
64const { getGlobalDispatcher } = require('../../global')
65const { webidl } = require('./webidl')
66const { STATUS_CODES } = require('node:http')
67const GET_OR_HEAD = ['GET', 'HEAD']
68
69const defaultUserAgent = typeof __UNDICI_IS_NODE__ !== 'undefined' || typeof esbuildDetection !== 'undefined'
70 ? 'node'
71 : 'undici'
72
73/** @type {import('buffer').resolveObjectURL} */
74let resolveObjectURL
75
76class Fetch extends EE {
77 constructor (dispatcher) {
78 super()
79
80 this.dispatcher = dispatcher
81 this.connection = null
82 this.dump = false
83 this.state = 'ongoing'
84 }
85
86 terminate (reason) {
87 if (this.state !== 'ongoing') {
88 return
89 }
90
91 this.state = 'terminated'
92 this.connection?.destroy(reason)
93 this.emit('terminated', reason)
94 }
95
96 // https://fetch.spec.whatwg.org/#fetch-controller-abort
97 abort (error) {
98 if (this.state !== 'ongoing') {
99 return
100 }
101
102 // 1. Set controller’s state to "aborted".
103 this.state = 'aborted'
104
105 // 2. Let fallbackError be an "AbortError" DOMException.
106 // 3. Set error to fallbackError if it is not given.
107 if (!error) {
108 error = new DOMException('The operation was aborted.', 'AbortError')
109 }
110
111 // 4. Let serializedError be StructuredSerialize(error).
112 // If that threw an exception, catch it, and let
113 // serializedError be StructuredSerialize(fallbackError).
114
115 // 5. Set controller’s serialized abort reason to serializedError.
116 this.serializedAbortReason = error
117
118 this.connection?.destroy(error)
119 this.emit('terminated', error)
120 }
121}
122
123function handleFetchDone (response) {
124 finalizeAndReportTiming(response, 'fetch')
125}
126
127// https://fetch.spec.whatwg.org/#fetch-method
128function fetch (input, init = undefined) {
129 webidl.argumentLengthCheck(arguments, 1, 'globalThis.fetch')
130
131 // 1. Let p be a new promise.
132 let p = createDeferredPromise()
133
134 // 2. Let requestObject be the result of invoking the initial value of
135 // Request as constructor with input and init as arguments. If this throws
136 // an exception, reject p with it and return p.
137 let requestObject
138
139 try {
140 requestObject = new Request(input, init)
141 } catch (e) {
142 p.reject(e)
143 return p.promise
144 }
145
146 // 3. Let request be requestObject’s request.
147 const request = requestObject[kState]
148
149 // 4. If requestObject’s signal’s aborted flag is set, then:
150 if (requestObject.signal.aborted) {
151 // 1. Abort the fetch() call with p, request, null, and
152 // requestObject’s signal’s abort reason.
153 abortFetch(p, request, null, requestObject.signal.reason)
154
155 // 2. Return p.
156 return p.promise
157 }
158
159 // 5. Let globalObject be request’s client’s global object.
160 const globalObject = request.client.globalObject
161
162 // 6. If globalObject is a ServiceWorkerGlobalScope object, then set
163 // request’s service-workers mode to "none".
164 if (globalObject?.constructor?.name === 'ServiceWorkerGlobalScope') {
165 request.serviceWorkers = 'none'
166 }
167
168 // 7. Let responseObject be null.
169 let responseObject = null
170
171 // 8. Let relevantRealm be this’s relevant Realm.
172
173 // 9. Let locallyAborted be false.
174 let locallyAborted = false
175
176 // 10. Let controller be null.
177 let controller = null
178
179 // 11. Add the following abort steps to requestObject’s signal:
180 addAbortListener(
181 requestObject.signal,
182 () => {
183 // 1. Set locallyAborted to true.
184 locallyAborted = true
185
186 // 2. Assert: controller is non-null.
187 assert(controller != null)
188
189 // 3. Abort controller with requestObject’s signal’s abort reason.
190 controller.abort(requestObject.signal.reason)
191
192 const realResponse = responseObject?.deref()
193
194 // 4. Abort the fetch() call with p, request, responseObject,
195 // and requestObject’s signal’s abort reason.
196 abortFetch(p, request, realResponse, requestObject.signal.reason)
197 }
198 )
199
200 // 12. Let handleFetchDone given response response be to finalize and
201 // report timing with response, globalObject, and "fetch".
202 // see function handleFetchDone
203
204 // 13. Set controller to the result of calling fetch given request,
205 // with processResponseEndOfBody set to handleFetchDone, and processResponse
206 // given response being these substeps:
207
208 const processResponse = (response) => {
209 // 1. If locallyAborted is true, terminate these substeps.
210 if (locallyAborted) {
211 return
212 }
213
214 // 2. If response’s aborted flag is set, then:
215 if (response.aborted) {
216 // 1. Let deserializedError be the result of deserialize a serialized
217 // abort reason given controller’s serialized abort reason and
218 // relevantRealm.
219
220 // 2. Abort the fetch() call with p, request, responseObject, and
221 // deserializedError.
222
223 abortFetch(p, request, responseObject, controller.serializedAbortReason)
224 return
225 }
226
227 // 3. If response is a network error, then reject p with a TypeError
228 // and terminate these substeps.
229 if (response.type === 'error') {
230 p.reject(new TypeError('fetch failed', { cause: response.error }))
231 return
232 }
233
234 // 4. Set responseObject to the result of creating a Response object,
235 // given response, "immutable", and relevantRealm.
236 responseObject = new WeakRef(fromInnerResponse(response, 'immutable'))
237
238 // 5. Resolve p with responseObject.
239 p.resolve(responseObject.deref())
240 p = null
241 }
242
243 controller = fetching({
244 request,
245 processResponseEndOfBody: handleFetchDone,
246 processResponse,
247 dispatcher: requestObject[kDispatcher] // undici
248 })
249
250 // 14. Return p.
251 return p.promise
252}
253
254// https://fetch.spec.whatwg.org/#finalize-and-report-timing
255function finalizeAndReportTiming (response, initiatorType = 'other') {
256 // 1. If response is an aborted network error, then return.
257 if (response.type === 'error' && response.aborted) {
258 return
259 }
260
261 // 2. If response’s URL list is null or empty, then return.
262 if (!response.urlList?.length) {
263 return
264 }
265
266 // 3. Let originalURL be response’s URL list[0].
267 const originalURL = response.urlList[0]
268
269 // 4. Let timingInfo be response’s timing info.
270 let timingInfo = response.timingInfo
271
272 // 5. Let cacheState be response’s cache state.
273 let cacheState = response.cacheState
274
275 // 6. If originalURL’s scheme is not an HTTP(S) scheme, then return.
276 if (!urlIsHttpHttpsScheme(originalURL)) {
277 return
278 }
279
280 // 7. If timingInfo is null, then return.
281 if (timingInfo === null) {
282 return
283 }
284
285 // 8. If response’s timing allow passed flag is not set, then:
286 if (!response.timingAllowPassed) {
287 // 1. Set timingInfo to a the result of creating an opaque timing info for timingInfo.
288 timingInfo = createOpaqueTimingInfo({
289 startTime: timingInfo.startTime
290 })
291
292 // 2. Set cacheState to the empty string.
293 cacheState = ''
294 }
295
296 // 9. Set timingInfo’s end time to the coarsened shared current time
297 // given global’s relevant settings object’s cross-origin isolated
298 // capability.
299 // TODO: given global’s relevant settings object’s cross-origin isolated
300 // capability?
301 timingInfo.endTime = coarsenedSharedCurrentTime()
302
303 // 10. Set response’s timing info to timingInfo.
304 response.timingInfo = timingInfo
305
306 // 11. Mark resource timing for timingInfo, originalURL, initiatorType,
307 // global, and cacheState.
308 markResourceTiming(
309 timingInfo,
310 originalURL.href,
311 initiatorType,
312 globalThis,
313 cacheState
314 )
315}
316
317// https://w3c.github.io/resource-timing/#dfn-mark-resource-timing
318const markResourceTiming = performance.markResourceTiming
319
320// https://fetch.spec.whatwg.org/#abort-fetch
321function abortFetch (p, request, responseObject, error) {
322 // 1. Reject promise with error.
323 if (p) {
324 // We might have already resolved the promise at this stage
325 p.reject(error)
326 }
327
328 // 2. If request’s body is not null and is readable, then cancel request’s
329 // body with error.
330 if (request.body != null && isReadable(request.body?.stream)) {
331 request.body.stream.cancel(error).catch((err) => {
332 if (err.code === 'ERR_INVALID_STATE') {
333 // Node bug?
334 return
335 }
336 throw err
337 })
338 }
339
340 // 3. If responseObject is null, then return.
341 if (responseObject == null) {
342 return
343 }
344
345 // 4. Let response be responseObject’s response.
346 const response = responseObject[kState]
347
348 // 5. If response’s body is not null and is readable, then error response’s
349 // body with error.
350 if (response.body != null && isReadable(response.body?.stream)) {
351 response.body.stream.cancel(error).catch((err) => {
352 if (err.code === 'ERR_INVALID_STATE') {
353 // Node bug?
354 return
355 }
356 throw err
357 })
358 }
359}
360
361// https://fetch.spec.whatwg.org/#fetching
362function fetching ({
363 request,
364 processRequestBodyChunkLength,
365 processRequestEndOfBody,
366 processResponse,
367 processResponseEndOfBody,
368 processResponseConsumeBody,
369 useParallelQueue = false,
370 dispatcher = getGlobalDispatcher() // undici
371}) {
372 // Ensure that the dispatcher is set accordingly
373 assert(dispatcher)
374
375 // 1. Let taskDestination be null.
376 let taskDestination = null
377
378 // 2. Let crossOriginIsolatedCapability be false.
379 let crossOriginIsolatedCapability = false
380
381 // 3. If request’s client is non-null, then:
382 if (request.client != null) {
383 // 1. Set taskDestination to request’s client’s global object.
384 taskDestination = request.client.globalObject
385
386 // 2. Set crossOriginIsolatedCapability to request’s client’s cross-origin
387 // isolated capability.
388 crossOriginIsolatedCapability =
389 request.client.crossOriginIsolatedCapability
390 }
391
392 // 4. If useParallelQueue is true, then set taskDestination to the result of
393 // starting a new parallel queue.
394 // TODO
395
396 // 5. Let timingInfo be a new fetch timing info whose start time and
397 // post-redirect start time are the coarsened shared current time given
398 // crossOriginIsolatedCapability.
399 const currentTime = coarsenedSharedCurrentTime(crossOriginIsolatedCapability)
400 const timingInfo = createOpaqueTimingInfo({
401 startTime: currentTime
402 })
403
404 // 6. Let fetchParams be a new fetch params whose
405 // request is request,
406 // timing info is timingInfo,
407 // process request body chunk length is processRequestBodyChunkLength,
408 // process request end-of-body is processRequestEndOfBody,
409 // process response is processResponse,
410 // process response consume body is processResponseConsumeBody,
411 // process response end-of-body is processResponseEndOfBody,
412 // task destination is taskDestination,
413 // and cross-origin isolated capability is crossOriginIsolatedCapability.
414 const fetchParams = {
415 controller: new Fetch(dispatcher),
416 request,
417 timingInfo,
418 processRequestBodyChunkLength,
419 processRequestEndOfBody,
420 processResponse,
421 processResponseConsumeBody,
422 processResponseEndOfBody,
423 taskDestination,
424 crossOriginIsolatedCapability
425 }
426
427 // 7. If request’s body is a byte sequence, then set request’s body to
428 // request’s body as a body.
429 // NOTE: Since fetching is only called from fetch, body should already be
430 // extracted.
431 assert(!request.body || request.body.stream)
432
433 // 8. If request’s window is "client", then set request’s window to request’s
434 // client, if request’s client’s global object is a Window object; otherwise
435 // "no-window".
436 if (request.window === 'client') {
437 // TODO: What if request.client is null?
438 request.window =
439 request.client?.globalObject?.constructor?.name === 'Window'
440 ? request.client
441 : 'no-window'
442 }
443
444 // 9. If request’s origin is "client", then set request’s origin to request’s
445 // client’s origin.
446 if (request.origin === 'client') {
447 request.origin = request.client.origin
448 }
449
450 // 10. If all of the following conditions are true:
451 // TODO
452
453 // 11. If request’s policy container is "client", then:
454 if (request.policyContainer === 'client') {
455 // 1. If request’s client is non-null, then set request’s policy
456 // container to a clone of request’s client’s policy container. [HTML]
457 if (request.client != null) {
458 request.policyContainer = clonePolicyContainer(
459 request.client.policyContainer
460 )
461 } else {
462 // 2. Otherwise, set request’s policy container to a new policy
463 // container.
464 request.policyContainer = makePolicyContainer()
465 }
466 }
467
468 // 12. If request’s header list does not contain `Accept`, then:
469 if (!request.headersList.contains('accept', true)) {
470 // 1. Let value be `*/*`.
471 const value = '*/*'
472
473 // 2. A user agent should set value to the first matching statement, if
474 // any, switching on request’s destination:
475 // "document"
476 // "frame"
477 // "iframe"
478 // `text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8`
479 // "image"
480 // `image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5`
481 // "style"
482 // `text/css,*/*;q=0.1`
483 // TODO
484
485 // 3. Append `Accept`/value to request’s header list.
486 request.headersList.append('accept', value, true)
487 }
488
489 // 13. If request’s header list does not contain `Accept-Language`, then
490 // user agents should append `Accept-Language`/an appropriate value to
491 // request’s header list.
492 if (!request.headersList.contains('accept-language', true)) {
493 request.headersList.append('accept-language', '*', true)
494 }
495
496 // 14. If request’s priority is null, then use request’s initiator and
497 // destination appropriately in setting request’s priority to a
498 // user-agent-defined object.
499 if (request.priority === null) {
500 // TODO
501 }
502
503 // 15. If request is a subresource request, then:
504 if (subresourceSet.has(request.destination)) {
505 // TODO
506 }
507
508 // 16. Run main fetch given fetchParams.
509 mainFetch(fetchParams)
510 .catch(err => {
511 fetchParams.controller.terminate(err)
512 })
513
514 // 17. Return fetchParam's controller
515 return fetchParams.controller
516}
517
518// https://fetch.spec.whatwg.org/#concept-main-fetch
519async function mainFetch (fetchParams, recursive = false) {
520 // 1. Let request be fetchParams’s request.
521 const request = fetchParams.request
522
523 // 2. Let response be null.
524 let response = null
525
526 // 3. If request’s local-URLs-only flag is set and request’s current URL is
527 // not local, then set response to a network error.
528 if (request.localURLsOnly && !urlIsLocal(requestCurrentURL(request))) {
529 response = makeNetworkError('local URLs only')
530 }
531
532 // 4. Run report Content Security Policy violations for request.
533 // TODO
534
535 // 5. Upgrade request to a potentially trustworthy URL, if appropriate.
536 tryUpgradeRequestToAPotentiallyTrustworthyURL(request)
537
538 // 6. If should request be blocked due to a bad port, should fetching request
539 // be blocked as mixed content, or should request be blocked by Content
540 // Security Policy returns blocked, then set response to a network error.
541 if (requestBadPort(request) === 'blocked') {
542 response = makeNetworkError('bad port')
543 }
544 // TODO: should fetching request be blocked as mixed content?
545 // TODO: should request be blocked by Content Security Policy?
546
547 // 7. If request’s referrer policy is the empty string, then set request’s
548 // referrer policy to request’s policy container’s referrer policy.
549 if (request.referrerPolicy === '') {
550 request.referrerPolicy = request.policyContainer.referrerPolicy
551 }
552
553 // 8. If request’s referrer is not "no-referrer", then set request’s
554 // referrer to the result of invoking determine request’s referrer.
555 if (request.referrer !== 'no-referrer') {
556 request.referrer = determineRequestsReferrer(request)
557 }
558
559 // 9. Set request’s current URL’s scheme to "https" if all of the following
560 // conditions are true:
561 // - request’s current URL’s scheme is "http"
562 // - request’s current URL’s host is a domain
563 // - Matching request’s current URL’s host per Known HSTS Host Domain Name
564 // Matching results in either a superdomain match with an asserted
565 // includeSubDomains directive or a congruent match (with or without an
566 // asserted includeSubDomains directive). [HSTS]
567 // TODO
568
569 // 10. If recursive is false, then run the remaining steps in parallel.
570 // TODO
571
572 // 11. If response is null, then set response to the result of running
573 // the steps corresponding to the first matching statement:
574 if (response === null) {
575 response = await (async () => {
576 const currentURL = requestCurrentURL(request)
577
578 if (
579 // - request’s current URL’s origin is same origin with request’s origin,
580 // and request’s response tainting is "basic"
581 (sameOrigin(currentURL, request.url) && request.responseTainting === 'basic') ||
582 // request’s current URL’s scheme is "data"
583 (currentURL.protocol === 'data:') ||
584 // - request’s mode is "navigate" or "websocket"
585 (request.mode === 'navigate' || request.mode === 'websocket')
586 ) {
587 // 1. Set request’s response tainting to "basic".
588 request.responseTainting = 'basic'
589
590 // 2. Return the result of running scheme fetch given fetchParams.
591 return await schemeFetch(fetchParams)
592 }
593
594 // request’s mode is "same-origin"
595 if (request.mode === 'same-origin') {
596 // 1. Return a network error.
597 return makeNetworkError('request mode cannot be "same-origin"')
598 }
599
600 // request’s mode is "no-cors"
601 if (request.mode === 'no-cors') {
602 // 1. If request’s redirect mode is not "follow", then return a network
603 // error.
604 if (request.redirect !== 'follow') {
605 return makeNetworkError(
606 'redirect mode cannot be "follow" for "no-cors" request'
607 )
608 }
609
610 // 2. Set request’s response tainting to "opaque".
611 request.responseTainting = 'opaque'
612
613 // 3. Return the result of running scheme fetch given fetchParams.
614 return await schemeFetch(fetchParams)
615 }
616
617 // request’s current URL’s scheme is not an HTTP(S) scheme
618 if (!urlIsHttpHttpsScheme(requestCurrentURL(request))) {
619 // Return a network error.
620 return makeNetworkError('URL scheme must be a HTTP(S) scheme')
621 }
622
623 // - request’s use-CORS-preflight flag is set
624 // - request’s unsafe-request flag is set and either request’s method is
625 // not a CORS-safelisted method or CORS-unsafe request-header names with
626 // request’s header list is not empty
627 // 1. Set request’s response tainting to "cors".
628 // 2. Let corsWithPreflightResponse be the result of running HTTP fetch
629 // given fetchParams and true.
630 // 3. If corsWithPreflightResponse is a network error, then clear cache
631 // entries using request.
632 // 4. Return corsWithPreflightResponse.
633 // TODO
634
635 // Otherwise
636 // 1. Set request’s response tainting to "cors".
637 request.responseTainting = 'cors'
638
639 // 2. Return the result of running HTTP fetch given fetchParams.
640 return await httpFetch(fetchParams)
641 })()
642 }
643
644 // 12. If recursive is true, then return response.
645 if (recursive) {
646 return response
647 }
648
649 // 13. If response is not a network error and response is not a filtered
650 // response, then:
651 if (response.status !== 0 && !response.internalResponse) {
652 // If request’s response tainting is "cors", then:
653 if (request.responseTainting === 'cors') {
654 // 1. Let headerNames be the result of extracting header list values
655 // given `Access-Control-Expose-Headers` and response’s header list.
656 // TODO
657 // 2. If request’s credentials mode is not "include" and headerNames
658 // contains `*`, then set response’s CORS-exposed header-name list to
659 // all unique header names in response’s header list.
660 // TODO
661 // 3. Otherwise, if headerNames is not null or failure, then set
662 // response’s CORS-exposed header-name list to headerNames.
663 // TODO
664 }
665
666 // Set response to the following filtered response with response as its
667 // internal response, depending on request’s response tainting:
668 if (request.responseTainting === 'basic') {
669 response = filterResponse(response, 'basic')
670 } else if (request.responseTainting === 'cors') {
671 response = filterResponse(response, 'cors')
672 } else if (request.responseTainting === 'opaque') {
673 response = filterResponse(response, 'opaque')
674 } else {
675 assert(false)
676 }
677 }
678
679 // 14. Let internalResponse be response, if response is a network error,
680 // and response’s internal response otherwise.
681 let internalResponse =
682 response.status === 0 ? response : response.internalResponse
683
684 // 15. If internalResponse’s URL list is empty, then set it to a clone of
685 // request’s URL list.
686 if (internalResponse.urlList.length === 0) {
687 internalResponse.urlList.push(...request.urlList)
688 }
689
690 // 16. If request’s timing allow failed flag is unset, then set
691 // internalResponse’s timing allow passed flag.
692 if (!request.timingAllowFailed) {
693 response.timingAllowPassed = true
694 }
695
696 // 17. If response is not a network error and any of the following returns
697 // blocked
698 // - should internalResponse to request be blocked as mixed content
699 // - should internalResponse to request be blocked by Content Security Policy
700 // - should internalResponse to request be blocked due to its MIME type
701 // - should internalResponse to request be blocked due to nosniff
702 // TODO
703
704 // 18. If response’s type is "opaque", internalResponse’s status is 206,
705 // internalResponse’s range-requested flag is set, and request’s header
706 // list does not contain `Range`, then set response and internalResponse
707 // to a network error.
708 if (
709 response.type === 'opaque' &&
710 internalResponse.status === 206 &&
711 internalResponse.rangeRequested &&
712 !request.headers.contains('range', true)
713 ) {
714 response = internalResponse = makeNetworkError()
715 }
716
717 // 19. If response is not a network error and either request’s method is
718 // `HEAD` or `CONNECT`, or internalResponse’s status is a null body status,
719 // set internalResponse’s body to null and disregard any enqueuing toward
720 // it (if any).
721 if (
722 response.status !== 0 &&
723 (request.method === 'HEAD' ||
724 request.method === 'CONNECT' ||
725 nullBodyStatus.includes(internalResponse.status))
726 ) {
727 internalResponse.body = null
728 fetchParams.controller.dump = true
729 }
730
731 // 20. If request’s integrity metadata is not the empty string, then:
732 if (request.integrity) {
733 // 1. Let processBodyError be this step: run fetch finale given fetchParams
734 // and a network error.
735 const processBodyError = (reason) =>
736 fetchFinale(fetchParams, makeNetworkError(reason))
737
738 // 2. If request’s response tainting is "opaque", or response’s body is null,
739 // then run processBodyError and abort these steps.
740 if (request.responseTainting === 'opaque' || response.body == null) {
741 processBodyError(response.error)
742 return
743 }
744
745 // 3. Let processBody given bytes be these steps:
746 const processBody = (bytes) => {
747 // 1. If bytes do not match request’s integrity metadata,
748 // then run processBodyError and abort these steps. [SRI]
749 if (!bytesMatch(bytes, request.integrity)) {
750 processBodyError('integrity mismatch')
751 return
752 }
753
754 // 2. Set response’s body to bytes as a body.
755 response.body = safelyExtractBody(bytes)[0]
756
757 // 3. Run fetch finale given fetchParams and response.
758 fetchFinale(fetchParams, response)
759 }
760
761 // 4. Fully read response’s body given processBody and processBodyError.
762 await fullyReadBody(response.body, processBody, processBodyError)
763 } else {
764 // 21. Otherwise, run fetch finale given fetchParams and response.
765 fetchFinale(fetchParams, response)
766 }
767}
768
769// https://fetch.spec.whatwg.org/#concept-scheme-fetch
770// given a fetch params fetchParams
771function schemeFetch (fetchParams) {
772 // Note: since the connection is destroyed on redirect, which sets fetchParams to a
773 // cancelled state, we do not want this condition to trigger *unless* there have been
774 // no redirects. See https://github.com/nodejs/undici/issues/1776
775 // 1. If fetchParams is canceled, then return the appropriate network error for fetchParams.
776 if (isCancelled(fetchParams) && fetchParams.request.redirectCount === 0) {
777 return Promise.resolve(makeAppropriateNetworkError(fetchParams))
778 }
779
780 // 2. Let request be fetchParams’s request.
781 const { request } = fetchParams
782
783 const { protocol: scheme } = requestCurrentURL(request)
784
785 // 3. Switch on request’s current URL’s scheme and run the associated steps:
786 switch (scheme) {
787 case 'about:': {
788 // If request’s current URL’s path is the string "blank", then return a new response
789 // whose status message is `OK`, header list is « (`Content-Type`, `text/html;charset=utf-8`) »,
790 // and body is the empty byte sequence as a body.
791
792 // Otherwise, return a network error.
793 return Promise.resolve(makeNetworkError('about scheme is not supported'))
794 }
795 case 'blob:': {
796 if (!resolveObjectURL) {
797 resolveObjectURL = require('node:buffer').resolveObjectURL
798 }
799
800 // 1. Let blobURLEntry be request’s current URL’s blob URL entry.
801 const blobURLEntry = requestCurrentURL(request)
802
803 // https://github.com/web-platform-tests/wpt/blob/7b0ebaccc62b566a1965396e5be7bb2bc06f841f/FileAPI/url/resources/fetch-tests.js#L52-L56
804 // Buffer.resolveObjectURL does not ignore URL queries.
805 if (blobURLEntry.search.length !== 0) {
806 return Promise.resolve(makeNetworkError('NetworkError when attempting to fetch resource.'))
807 }
808
809 const blob = resolveObjectURL(blobURLEntry.toString())
810
811 // 2. If request’s method is not `GET`, blobURLEntry is null, or blobURLEntry’s
812 // object is not a Blob object, then return a network error.
813 if (request.method !== 'GET' || !isBlobLike(blob)) {
814 return Promise.resolve(makeNetworkError('invalid method'))
815 }
816
817 // 3. Let blob be blobURLEntry’s object.
818 // Note: done above
819
820 // 4. Let response be a new response.
821 const response = makeResponse()
822
823 // 5. Let fullLength be blob’s size.
824 const fullLength = blob.size
825
826 // 6. Let serializedFullLength be fullLength, serialized and isomorphic encoded.
827 const serializedFullLength = isomorphicEncode(`${fullLength}`)
828
829 // 7. Let type be blob’s type.
830 const type = blob.type
831
832 // 8. If request’s header list does not contain `Range`:
833 // 9. Otherwise:
834 if (!request.headersList.contains('range', true)) {
835 // 1. Let bodyWithType be the result of safely extracting blob.
836 // Note: in the FileAPI a blob "object" is a Blob *or* a MediaSource.
837 // In node, this can only ever be a Blob. Therefore we can safely
838 // use extractBody directly.
839 const bodyWithType = extractBody(blob)
840
841 // 2. Set response’s status message to `OK`.
842 response.statusText = 'OK'
843
844 // 3. Set response’s body to bodyWithType’s body.
845 response.body = bodyWithType[0]
846
847 // 4. Set response’s header list to « (`Content-Length`, serializedFullLength), (`Content-Type`, type) ».
848 response.headersList.set('content-length', serializedFullLength, true)
849 response.headersList.set('content-type', type, true)
850 } else {
851 // 1. Set response’s range-requested flag.
852 response.rangeRequested = true
853
854 // 2. Let rangeHeader be the result of getting `Range` from request’s header list.
855 const rangeHeader = request.headersList.get('range', true)
856
857 // 3. Let rangeValue be the result of parsing a single range header value given rangeHeader and true.
858 const rangeValue = simpleRangeHeaderValue(rangeHeader, true)
859
860 // 4. If rangeValue is failure, then return a network error.
861 if (rangeValue === 'failure') {
862 return Promise.resolve(makeNetworkError('failed to fetch the data URL'))
863 }
864
865 // 5. Let (rangeStart, rangeEnd) be rangeValue.
866 let { rangeStartValue: rangeStart, rangeEndValue: rangeEnd } = rangeValue
867
868 // 6. If rangeStart is null:
869 // 7. Otherwise:
870 if (rangeStart === null) {
871 // 1. Set rangeStart to fullLength − rangeEnd.
872 rangeStart = fullLength - rangeEnd
873
874 // 2. Set rangeEnd to rangeStart + rangeEnd − 1.
875 rangeEnd = rangeStart + rangeEnd - 1
876 } else {
877 // 1. If rangeStart is greater than or equal to fullLength, then return a network error.
878 if (rangeStart >= fullLength) {
879 return Promise.resolve(makeNetworkError('Range start is greater than the blob\'s size.'))
880 }
881
882 // 2. If rangeEnd is null or rangeEnd is greater than or equal to fullLength, then set
883 // rangeEnd to fullLength − 1.
884 if (rangeEnd === null || rangeEnd >= fullLength) {
885 rangeEnd = fullLength - 1
886 }
887 }
888
889 // 8. Let slicedBlob be the result of invoking slice blob given blob, rangeStart,
890 // rangeEnd + 1, and type.
891 const slicedBlob = blob.slice(rangeStart, rangeEnd, type)
892
893 // 9. Let slicedBodyWithType be the result of safely extracting slicedBlob.
894 // Note: same reason as mentioned above as to why we use extractBody
895 const slicedBodyWithType = extractBody(slicedBlob)
896
897 // 10. Set response’s body to slicedBodyWithType’s body.
898 response.body = slicedBodyWithType[0]
899
900 // 11. Let serializedSlicedLength be slicedBlob’s size, serialized and isomorphic encoded.
901 const serializedSlicedLength = isomorphicEncode(`${slicedBlob.size}`)
902
903 // 12. Let contentRange be the result of invoking build a content range given rangeStart,
904 // rangeEnd, and fullLength.
905 const contentRange = buildContentRange(rangeStart, rangeEnd, fullLength)
906
907 // 13. Set response’s status to 206.
908 response.status = 206
909
910 // 14. Set response’s status message to `Partial Content`.
911 response.statusText = 'Partial Content'
912
913 // 15. Set response’s header list to « (`Content-Length`, serializedSlicedLength),
914 // (`Content-Type`, type), (`Content-Range`, contentRange) ».
915 response.headersList.set('content-length', serializedSlicedLength, true)
916 response.headersList.set('content-type', type, true)
917 response.headersList.set('content-range', contentRange, true)
918 }
919
920 // 10. Return response.
921 return Promise.resolve(response)
922 }
923 case 'data:': {
924 // 1. Let dataURLStruct be the result of running the
925 // data: URL processor on request’s current URL.
926 const currentURL = requestCurrentURL(request)
927 const dataURLStruct = dataURLProcessor(currentURL)
928
929 // 2. If dataURLStruct is failure, then return a
930 // network error.
931 if (dataURLStruct === 'failure') {
932 return Promise.resolve(makeNetworkError('failed to fetch the data URL'))
933 }
934
935 // 3. Let mimeType be dataURLStruct’s MIME type, serialized.
936 const mimeType = serializeAMimeType(dataURLStruct.mimeType)
937
938 // 4. Return a response whose status message is `OK`,
939 // header list is « (`Content-Type`, mimeType) »,
940 // and body is dataURLStruct’s body as a body.
941 return Promise.resolve(makeResponse({
942 statusText: 'OK',
943 headersList: [
944 ['content-type', { name: 'Content-Type', value: mimeType }]
945 ],
946 body: safelyExtractBody(dataURLStruct.body)[0]
947 }))
948 }
949 case 'file:': {
950 // For now, unfortunate as it is, file URLs are left as an exercise for the reader.
951 // When in doubt, return a network error.
952 return Promise.resolve(makeNetworkError('not implemented... yet...'))
953 }
954 case 'http:':
955 case 'https:': {
956 // Return the result of running HTTP fetch given fetchParams.
957
958 return httpFetch(fetchParams)
959 .catch((err) => makeNetworkError(err))
960 }
961 default: {
962 return Promise.resolve(makeNetworkError('unknown scheme'))
963 }
964 }
965}
966
967// https://fetch.spec.whatwg.org/#finalize-response
968function finalizeResponse (fetchParams, response) {
969 // 1. Set fetchParams’s request’s done flag.
970 fetchParams.request.done = true
971
972 // 2, If fetchParams’s process response done is not null, then queue a fetch
973 // task to run fetchParams’s process response done given response, with
974 // fetchParams’s task destination.
975 if (fetchParams.processResponseDone != null) {
976 queueMicrotask(() => fetchParams.processResponseDone(response))
977 }
978}
979
980// https://fetch.spec.whatwg.org/#fetch-finale
981function fetchFinale (fetchParams, response) {
982 // 1. Let timingInfo be fetchParams’s timing info.
983 let timingInfo = fetchParams.timingInfo
984
985 // 2. If response is not a network error and fetchParams’s request’s client is a secure context,
986 // then set timingInfo’s server-timing headers to the result of getting, decoding, and splitting
987 // `Server-Timing` from response’s internal response’s header list.
988 // TODO
989
990 // 3. Let processResponseEndOfBody be the following steps:
991 const processResponseEndOfBody = () => {
992 // 1. Let unsafeEndTime be the unsafe shared current time.
993 const unsafeEndTime = Date.now() // ?
994
995 // 2. If fetchParams’s request’s destination is "document", then set fetchParams’s controller’s
996 // full timing info to fetchParams’s timing info.
997 if (fetchParams.request.destination === 'document') {
998 fetchParams.controller.fullTimingInfo = timingInfo
999 }
1000
1001 // 3. Set fetchParams’s controller’s report timing steps to the following steps given a global object global:
1002 fetchParams.controller.reportTimingSteps = () => {
1003 // 1. If fetchParams’s request’s URL’s scheme is not an HTTP(S) scheme, then return.
1004 if (fetchParams.request.url.protocol !== 'https:') {
1005 return
1006 }
1007
1008 // 2. Set timingInfo’s end time to the relative high resolution time given unsafeEndTime and global.
1009 timingInfo.endTime = unsafeEndTime
1010
1011 // 3. Let cacheState be response’s cache state.
1012 let cacheState = response.cacheState
1013
1014 // 4. Let bodyInfo be response’s body info.
1015 const bodyInfo = response.bodyInfo
1016
1017 // 5. If response’s timing allow passed flag is not set, then set timingInfo to the result of creating an
1018 // opaque timing info for timingInfo and set cacheState to the empty string.
1019 if (!response.timingAllowPassed) {
1020 timingInfo = createOpaqueTimingInfo(timingInfo)
1021
1022 cacheState = ''
1023 }
1024
1025 // 6. Let responseStatus be 0.
1026 let responseStatus = 0
1027
1028 // 7. If fetchParams’s request’s mode is not "navigate" or response’s has-cross-origin-redirects is false:
1029 if (fetchParams.request.mode !== 'navigator' || !response.hasCrossOriginRedirects) {
1030 // 1. Set responseStatus to response’s status.
1031 responseStatus = response.status
1032
1033 // 2. Let mimeType be the result of extracting a MIME type from response’s header list.
1034 const mimeType = extractMimeType(response.headersList)
1035
1036 // 3. If mimeType is not failure, then set bodyInfo’s content type to the result of minimizing a supported MIME type given mimeType.
1037 if (mimeType !== 'failure') {
1038 bodyInfo.contentType = minimizeSupportedMimeType(mimeType)
1039 }
1040 }
1041
1042 // 8. If fetchParams’s request’s initiator type is non-null, then mark resource timing given timingInfo,
1043 // fetchParams’s request’s URL, fetchParams’s request’s initiator type, global, cacheState, bodyInfo,
1044 // and responseStatus.
1045 if (fetchParams.request.initiatorType != null) {
1046 // TODO: update markresourcetiming
1047 markResourceTiming(timingInfo, fetchParams.request.url.href, fetchParams.request.initiatorType, globalThis, cacheState, bodyInfo, responseStatus)
1048 }
1049 }
1050
1051 // 4. Let processResponseEndOfBodyTask be the following steps:
1052 const processResponseEndOfBodyTask = () => {
1053 // 1. Set fetchParams’s request’s done flag.
1054 fetchParams.request.done = true
1055
1056 // 2. If fetchParams’s process response end-of-body is non-null, then run fetchParams’s process
1057 // response end-of-body given response.
1058 if (fetchParams.processResponseEndOfBody != null) {
1059 queueMicrotask(() => fetchParams.processResponseEndOfBody(response))
1060 }
1061
1062 // 3. If fetchParams’s request’s initiator type is non-null and fetchParams’s request’s client’s
1063 // global object is fetchParams’s task destination, then run fetchParams’s controller’s report
1064 // timing steps given fetchParams’s request’s client’s global object.
1065 if (fetchParams.request.initiatorType != null) {
1066 fetchParams.controller.reportTimingSteps()
1067 }
1068 }
1069
1070 // 5. Queue a fetch task to run processResponseEndOfBodyTask with fetchParams’s task destination
1071 queueMicrotask(() => processResponseEndOfBodyTask())
1072 }
1073
1074 // 4. If fetchParams’s process response is non-null, then queue a fetch task to run fetchParams’s
1075 // process response given response, with fetchParams’s task destination.
1076 if (fetchParams.processResponse != null) {
1077 queueMicrotask(() => {
1078 fetchParams.processResponse(response)
1079 fetchParams.processResponse = null
1080 })
1081 }
1082
1083 // 5. Let internalResponse be response, if response is a network error; otherwise response’s internal response.
1084 const internalResponse = response.type === 'error' ? response : (response.internalResponse ?? response)
1085
1086 // 6. If internalResponse’s body is null, then run processResponseEndOfBody.
1087 // 7. Otherwise:
1088 if (internalResponse.body == null) {
1089 processResponseEndOfBody()
1090 } else {
1091 // mcollina: all the following steps of the specs are skipped.
1092 // The internal transform stream is not needed.
1093 // See https://github.com/nodejs/undici/pull/3093#issuecomment-2050198541
1094
1095 // 1. Let transformStream be a new TransformStream.
1096 // 2. Let identityTransformAlgorithm be an algorithm which, given chunk, enqueues chunk in transformStream.
1097 // 3. Set up transformStream with transformAlgorithm set to identityTransformAlgorithm and flushAlgorithm
1098 // set to processResponseEndOfBody.
1099 // 4. Set internalResponse’s body’s stream to the result of internalResponse’s body’s stream piped through transformStream.
1100
1101 finished(internalResponse.body.stream, () => {
1102 processResponseEndOfBody()
1103 })
1104 }
1105}
1106
1107// https://fetch.spec.whatwg.org/#http-fetch
1108async function httpFetch (fetchParams) {
1109 // 1. Let request be fetchParams’s request.
1110 const request = fetchParams.request
1111
1112 // 2. Let response be null.
1113 let response = null
1114
1115 // 3. Let actualResponse be null.
1116 let actualResponse = null
1117
1118 // 4. Let timingInfo be fetchParams’s timing info.
1119 const timingInfo = fetchParams.timingInfo
1120
1121 // 5. If request’s service-workers mode is "all", then:
1122 if (request.serviceWorkers === 'all') {
1123 // TODO
1124 }
1125
1126 // 6. If response is null, then:
1127 if (response === null) {
1128 // 1. If makeCORSPreflight is true and one of these conditions is true:
1129 // TODO
1130
1131 // 2. If request’s redirect mode is "follow", then set request’s
1132 // service-workers mode to "none".
1133 if (request.redirect === 'follow') {
1134 request.serviceWorkers = 'none'
1135 }
1136
1137 // 3. Set response and actualResponse to the result of running
1138 // HTTP-network-or-cache fetch given fetchParams.
1139 actualResponse = response = await httpNetworkOrCacheFetch(fetchParams)
1140
1141 // 4. If request’s response tainting is "cors" and a CORS check
1142 // for request and response returns failure, then return a network error.
1143 if (
1144 request.responseTainting === 'cors' &&
1145 corsCheck(request, response) === 'failure'
1146 ) {
1147 return makeNetworkError('cors failure')
1148 }
1149
1150 // 5. If the TAO check for request and response returns failure, then set
1151 // request’s timing allow failed flag.
1152 if (TAOCheck(request, response) === 'failure') {
1153 request.timingAllowFailed = true
1154 }
1155 }
1156
1157 // 7. If either request’s response tainting or response’s type
1158 // is "opaque", and the cross-origin resource policy check with
1159 // request’s origin, request’s client, request’s destination,
1160 // and actualResponse returns blocked, then return a network error.
1161 if (
1162 (request.responseTainting === 'opaque' || response.type === 'opaque') &&
1163 crossOriginResourcePolicyCheck(
1164 request.origin,
1165 request.client,
1166 request.destination,
1167 actualResponse
1168 ) === 'blocked'
1169 ) {
1170 return makeNetworkError('blocked')
1171 }
1172
1173 // 8. If actualResponse’s status is a redirect status, then:
1174 if (redirectStatusSet.has(actualResponse.status)) {
1175 // 1. If actualResponse’s status is not 303, request’s body is not null,
1176 // and the connection uses HTTP/2, then user agents may, and are even
1177 // encouraged to, transmit an RST_STREAM frame.
1178 // See, https://github.com/whatwg/fetch/issues/1288
1179 if (request.redirect !== 'manual') {
1180 fetchParams.controller.connection.destroy(undefined, false)
1181 }
1182
1183 // 2. Switch on request’s redirect mode:
1184 if (request.redirect === 'error') {
1185 // Set response to a network error.
1186 response = makeNetworkError('unexpected redirect')
1187 } else if (request.redirect === 'manual') {
1188 // Set response to an opaque-redirect filtered response whose internal
1189 // response is actualResponse.
1190 // NOTE(spec): On the web this would return an `opaqueredirect` response,
1191 // but that doesn't make sense server side.
1192 // See https://github.com/nodejs/undici/issues/1193.
1193 response = actualResponse
1194 } else if (request.redirect === 'follow') {
1195 // Set response to the result of running HTTP-redirect fetch given
1196 // fetchParams and response.
1197 response = await httpRedirectFetch(fetchParams, response)
1198 } else {
1199 assert(false)
1200 }