codekingpro/portable-devtools

1'use strict'
2const { Minipass } = require('minipass')
3const { MinipassSized } = require('minipass-sized')
4
5const Blob = require('./blob.js')
6const { BUFFER } = Blob
7const FetchError = require('./fetch-error.js')
8
9// optional dependency on 'iconv-lite'
10let decode
11try {
12  decode = require('iconv-lite').decode
13} catch (e) {
14  // defer error until textConverted is called
15}
16
17const INTERNALS = Symbol('Body internals')
18const CONSUME_BODY = Symbol('consumeBody')
19
20class Body {
21  constructor (bodyArg, options = {}) {
22    const { size = 0, timeout = 0 } = options
23    const body = bodyArg === undefined || bodyArg === null ? null
24      : isURLSearchParams(bodyArg) ? Buffer.from(bodyArg.toString())
25      : isBlob(bodyArg) ? bodyArg
26      : Buffer.isBuffer(bodyArg) ? bodyArg
27      : Object.prototype.toString.call(bodyArg) === '[object ArrayBuffer]'
28        ? Buffer.from(bodyArg)
29        : ArrayBuffer.isView(bodyArg)
30          ? Buffer.from(bodyArg.buffer, bodyArg.byteOffset, bodyArg.byteLength)
31          : Minipass.isStream(bodyArg) ? bodyArg
32          : Buffer.from(String(bodyArg))
33
34    this[INTERNALS] = {
35      body,
36      disturbed: false,
37      error: null,
38    }
39
40    this.size = size
41    this.timeout = timeout
42
43    if (Minipass.isStream(body)) {
44      body.on('error', er => {
45        const error = er.name === 'AbortError' ? er
46          : new FetchError(`Invalid response while trying to fetch ${
47            this.url}: ${er.message}`, 'system', er)
48        this[INTERNALS].error = error
49      })
50    }
51  }
52
53  get body () {
54    return this[INTERNALS].body
55  }
56
57  get bodyUsed () {
58    return this[INTERNALS].disturbed
59  }
60
61  arrayBuffer () {
62    return this[CONSUME_BODY]().then(buf =>
63      buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength))
64  }
65
66  blob () {
67    const ct = this.headers && this.headers.get('content-type') || ''
68    return this[CONSUME_BODY]().then(buf => Object.assign(
69      new Blob([], { type: ct.toLowerCase() }),
70      { [BUFFER]: buf }
71    ))
72  }
73
74  async json () {
75    const buf = await this[CONSUME_BODY]()
76    try {
77      return JSON.parse(buf.toString())
78    } catch (er) {
79      throw new FetchError(
80        `invalid json response body at ${this.url} reason: ${er.message}`,
81        'invalid-json'
82      )
83    }
84  }
85
86  text () {
87    return this[CONSUME_BODY]().then(buf => buf.toString())
88  }
89
90  buffer () {
91    return this[CONSUME_BODY]()
92  }
93
94  textConverted () {
95    /* istanbul ignore if */
96    if (typeof decode !== 'function') {
97      throw new Error('The package `iconv-lite` must be installed to use the textConverted() function')
98    }
99    return this[CONSUME_BODY]().then(buf => convertBody(buf, this.headers))
100  }
101
102  [CONSUME_BODY] () {
103    if (this[INTERNALS].disturbed) {
104      return Promise.reject(new TypeError(`body used already for: ${
105        this.url}`))
106    }
107
108    this[INTERNALS].disturbed = true
109
110    if (this[INTERNALS].error) {
111      return Promise.reject(this[INTERNALS].error)
112    }
113
114    // body is null
115    if (this.body === null) {
116      return Promise.resolve(Buffer.alloc(0))
117    }
118
119    if (Buffer.isBuffer(this.body)) {
120      return Promise.resolve(this.body)
121    }
122
123    const upstream = isBlob(this.body) ? this.body.stream() : this.body
124
125    /* istanbul ignore if: should never happen */
126    if (!Minipass.isStream(upstream)) {
127      return Promise.resolve(Buffer.alloc(0))
128    }
129
130    const stream = this.size && upstream instanceof MinipassSized ? upstream
131      : !this.size && upstream instanceof Minipass &&
132        !(upstream instanceof MinipassSized) ? upstream
133      : this.size ? new MinipassSized({ size: this.size })
134      : new Minipass()
135
136    // allow timeout on slow response body, but only if the stream is still writable. this
137    // makes the timeout center on the socket stream from lib/index.js rather than the
138    // intermediary minipass stream we create to receive the data
139    const resTimeout = this.timeout && stream.writable ? setTimeout(() => {
140      stream.emit('error', new FetchError(
141        `Response timeout while trying to fetch ${
142          this.url} (over ${this.timeout}ms)`, 'body-timeout'))
143    }, this.timeout) : null
144
145    // do not keep the process open just for this timeout, even
146    // though we expect it'll get cleared eventually.
147    if (resTimeout && resTimeout.unref) {
148      resTimeout.unref()
149    }
150
151    // do the pipe in the promise, because the pipe() can send too much
152    // data through right away and upset the MP Sized object
153    return new Promise((resolve) => {
154      // if the stream is some other kind of stream, then pipe through a MP
155      // so we can collect it more easily.
156      if (stream !== upstream) {
157        upstream.on('error', er => stream.emit('error', er))
158        upstream.pipe(stream)
159      }
160      resolve()
161    }).then(() => stream.concat()).then(buf => {
162      clearTimeout(resTimeout)
163      return buf
164    }).catch(er => {
165      clearTimeout(resTimeout)
166      // request was aborted, reject with this Error
167      if (er.name === 'AbortError' || er.name === 'FetchError') {
168        throw er
169      } else if (er.name === 'RangeError') {
170        throw new FetchError(`Could not create Buffer from response body for ${
171          this.url}: ${er.message}`, 'system', er)
172      } else {
173        // other errors, such as incorrect content-encoding or content-length
174        throw new FetchError(`Invalid response body while trying to fetch ${
175          this.url}: ${er.message}`, 'system', er)
176      }
177    })
178  }
179
180  static clone (instance) {
181    if (instance.bodyUsed) {
182      throw new Error('cannot clone body after it is used')
183    }
184
185    const body = instance.body
186
187    // check that body is a stream and not form-data object
188    // NB: can't clone the form-data object without having it as a dependency
189    if (Minipass.isStream(body) && typeof body.getBoundary !== 'function') {
190      // create a dedicated tee stream so that we don't lose data
191      // potentially sitting in the body stream's buffer by writing it
192      // immediately to p1 and not having it for p2.
193      const tee = new Minipass()
194      const p1 = new Minipass()
195      const p2 = new Minipass()
196      tee.on('error', er => {
197        p1.emit('error', er)
198        p2.emit('error', er)
199      })
200      body.on('error', er => tee.emit('error', er))
201      tee.pipe(p1)
202      tee.pipe(p2)
203      body.pipe(tee)
204      // set instance body to one fork, return the other
205      instance[INTERNALS].body = p1
206      return p2
207    } else {
208      return instance.body
209    }
210  }
211
212  static extractContentType (body) {
213    return body === null || body === undefined ? null
214      : typeof body === 'string' ? 'text/plain;charset=UTF-8'
215      : isURLSearchParams(body)
216        ? 'application/x-www-form-urlencoded;charset=UTF-8'
217        : isBlob(body) ? body.type || null
218        : Buffer.isBuffer(body) ? null
219        : Object.prototype.toString.call(body) === '[object ArrayBuffer]' ? null
220        : ArrayBuffer.isView(body) ? null
221        : typeof body.getBoundary === 'function'
222          ? `multipart/form-data;boundary=${body.getBoundary()}`
223          : Minipass.isStream(body) ? null
224          : 'text/plain;charset=UTF-8'
225  }
226
227  static getTotalBytes (instance) {
228    const { body } = instance
229    return (body === null || body === undefined) ? 0
230      : isBlob(body) ? body.size
231      : Buffer.isBuffer(body) ? body.length
232      : body && typeof body.getLengthSync === 'function' && (
233        // detect form data input from form-data module
234        body._lengthRetrievers &&
235        /* istanbul ignore next */ body._lengthRetrievers.length === 0 || // 1.x
236        body.hasKnownLength && body.hasKnownLength()) // 2.x
237        ? body.getLengthSync()
238        : null
239  }
240
241  static writeToStream (dest, instance) {
242    const { body } = instance
243
244    if (body === null || body === undefined) {
245      dest.end()
246    } else if (Buffer.isBuffer(body) || typeof body === 'string') {
247      dest.end(body)
248    } else {
249      // body is stream or blob
250      const stream = isBlob(body) ? body.stream() : body
251      stream.on('error', er => dest.emit('error', er)).pipe(dest)
252    }
253
254    return dest
255  }
256}
257
258Object.defineProperties(Body.prototype, {
259  body: { enumerable: true },
260  bodyUsed: { enumerable: true },
261  arrayBuffer: { enumerable: true },
262  blob: { enumerable: true },
263  json: { enumerable: true },
264  text: { enumerable: true },
265})
266
267const isURLSearchParams = obj =>
268  // Duck-typing as a necessary condition.
269  (typeof obj !== 'object' ||
270    typeof obj.append !== 'function' ||
271    typeof obj.delete !== 'function' ||
272    typeof obj.get !== 'function' ||
273    typeof obj.getAll !== 'function' ||
274    typeof obj.has !== 'function' ||
275    typeof obj.set !== 'function') ? false
276  // Brand-checking and more duck-typing as optional condition.
277  : obj.constructor.name === 'URLSearchParams' ||
278    Object.prototype.toString.call(obj) === '[object URLSearchParams]' ||
279    typeof obj.sort === 'function'
280
281const isBlob = obj =>
282  typeof obj === 'object' &&
283  typeof obj.arrayBuffer === 'function' &&
284  typeof obj.type === 'string' &&
285  typeof obj.stream === 'function' &&
286  typeof obj.constructor === 'function' &&
287  typeof obj.constructor.name === 'string' &&
288  /^(Blob|File)$/.test(obj.constructor.name) &&
289  /^(Blob|File)$/.test(obj[Symbol.toStringTag])
290
291const convertBody = (buffer, headers) => {
292  const ct = headers && headers.get('content-type')
293  let charset = 'utf-8'
294  let res
295
296  // header
297  if (ct) {
298    res = /charset=([^;]*)/i.exec(ct)
299  }
300
301  // no charset in content type, peek at response body for at most 1024 bytes
302  const str = buffer.slice(0, 1024).toString()
303
304  // html5
305  if (!res && str) {
306    res = /<meta.+?charset=(['"])(.+?)\1/i.exec(str)
307  }
308
309  // html4
310  if (!res && str) {
311    res = /<meta[\s]+?http-equiv=(['"])content-type\1[\s]+?content=(['"])(.+?)\2/i.exec(str)
312
313    if (!res) {
314      res = /<meta[\s]+?content=(['"])(.+?)\1[\s]+?http-equiv=(['"])content-type\3/i.exec(str)
315      if (res) {
316        res.pop()
317      } // drop last quote
318    }
319
320    if (res) {
321      res = /charset=(.*)/i.exec(res.pop())
322    }
323  }
324
325  // xml
326  if (!res && str) {
327    res = /<\?xml.+?encoding=(['"])(.+?)\1/i.exec(str)
328  }
329
330  // found charset
331  if (res) {
332    charset = res.pop()
333
334    // prevent decode issues when sites use incorrect encoding
335    // ref: https://hsivonen.fi/encoding-menu/
336    if (charset === 'gb2312' || charset === 'gbk') {
337      charset = 'gb18030'
338    }
339  }
340
341  if (charset === 'UTF-8') {
342    return buffer.toString('UTF-8')
343  }
344
345  charset = charset.toString().trim()
346    .replace(/^latin[-_]?(\d+)$/i, 'ISO-8859-$1')
347    .replace(/^win(?:dows)?[-_]?(\d+)$/i, 'WINDOWS-$1')
348    .replace(/^utf[-_]?(\d+)$/i, 'UTF-$1')
349    .replace(/^ks_c_5601-1987$/i, 'CP949')
350    .replace(/^us[-_]?ascii$/i, 'ASCII')
351    .toUpperCase()
352  try {
353    return decode(buffer, charset).toString()
354  } catch {
355    /* istanbul ignore next */
356    return buffer.toString('UTF-8')
357  }
358}
359
360module.exports = Body
361