codekingpro/portable-devtools

1"use strict"
2
3var Buffer = require("safer-buffer").Buffer
4
5// == UTF32-LE/BE codec. ==========================================================
6
7exports._utf32 = Utf32Codec
8
9function Utf32Codec (codecOptions, iconv) {
10  this.iconv = iconv
11  this.bomAware = true
12  this.isLE = codecOptions.isLE
13}
14
15exports.utf32le = { type: "_utf32", isLE: true }
16exports.utf32be = { type: "_utf32", isLE: false }
17
18// Aliases
19exports.ucs4le = "utf32le"
20exports.ucs4be = "utf32be"
21
22Utf32Codec.prototype.encoder = Utf32Encoder
23Utf32Codec.prototype.decoder = Utf32Decoder
24
25// -- Encoding
26
27function Utf32Encoder (options, codec) {
28  this.isLE = codec.isLE
29  this.highSurrogate = 0
30}
31
32Utf32Encoder.prototype.write = function (str) {
33  var src = Buffer.from(str, "ucs2")
34  var dst = Buffer.alloc(src.length * 2)
35  var write32 = this.isLE ? dst.writeUInt32LE : dst.writeUInt32BE
36  var offset = 0
37
38  for (var i = 0; i < src.length; i += 2) {
39    var code = src.readUInt16LE(i)
40    var isHighSurrogate = (code >= 0xD800 && code < 0xDC00)
41    var isLowSurrogate = (code >= 0xDC00 && code < 0xE000)
42
43    if (this.highSurrogate) {
44      if (isHighSurrogate || !isLowSurrogate) {
45        // There shouldn't be two high surrogates in a row, nor a high surrogate which isn't followed by a low
46        // surrogate. If this happens, keep the pending high surrogate as a stand-alone semi-invalid character
47        // (technically wrong, but expected by some applications, like Windows file names).
48        write32.call(dst, this.highSurrogate, offset)
49        offset += 4
50      } else {
51        // Create 32-bit value from high and low surrogates;
52        var codepoint = (((this.highSurrogate - 0xD800) << 10) | (code - 0xDC00)) + 0x10000
53
54        write32.call(dst, codepoint, offset)
55        offset += 4
56        this.highSurrogate = 0
57
58        continue
59      }
60    }
61
62    if (isHighSurrogate) { this.highSurrogate = code } else {
63      // Even if the current character is a low surrogate, with no previous high surrogate, we'll
64      // encode it as a semi-invalid stand-alone character for the same reasons expressed above for
65      // unpaired high surrogates.
66      write32.call(dst, code, offset)
67      offset += 4
68      this.highSurrogate = 0
69    }
70  }
71
72  if (offset < dst.length) { dst = dst.slice(0, offset) }
73
74  return dst
75}
76
77Utf32Encoder.prototype.end = function () {
78  // Treat any leftover high surrogate as a semi-valid independent character.
79  if (!this.highSurrogate) { return }
80
81  var buf = Buffer.alloc(4)
82
83  if (this.isLE) { buf.writeUInt32LE(this.highSurrogate, 0) } else { buf.writeUInt32BE(this.highSurrogate, 0) }
84
85  this.highSurrogate = 0
86
87  return buf
88}
89
90// -- Decoding
91
92function Utf32Decoder (options, codec) {
93  this.isLE = codec.isLE
94  this.badChar = codec.iconv.defaultCharUnicode.charCodeAt(0)
95  this.overflow = []
96}
97
98Utf32Decoder.prototype.write = function (src) {
99  if (src.length === 0) { return "" }
100
101  var i = 0
102  var codepoint = 0
103  var dst = Buffer.alloc(src.length + 4)
104  var offset = 0
105  var isLE = this.isLE
106  var overflow = this.overflow
107  var badChar = this.badChar
108
109  if (overflow.length > 0) {
110    for (; i < src.length && overflow.length < 4; i++) { overflow.push(src[i]) }
111
112    if (overflow.length === 4) {
113      // NOTE: codepoint is a signed int32 and can be negative.
114      // NOTE: We copied this block from below to help V8 optimize it (it works with array, not buffer).
115      if (isLE) {
116        codepoint = overflow[i] | (overflow[i + 1] << 8) | (overflow[i + 2] << 16) | (overflow[i + 3] << 24)
117      } else {
118        codepoint = overflow[i + 3] | (overflow[i + 2] << 8) | (overflow[i + 1] << 16) | (overflow[i] << 24)
119      }
120      overflow.length = 0
121
122      offset = _writeCodepoint(dst, offset, codepoint, badChar)
123    }
124  }
125
126  // Main loop. Should be as optimized as possible.
127  for (; i < src.length - 3; i += 4) {
128    // NOTE: codepoint is a signed int32 and can be negative.
129    if (isLE) {
130      codepoint = src[i] | (src[i + 1] << 8) | (src[i + 2] << 16) | (src[i + 3] << 24)
131    } else {
132      codepoint = src[i + 3] | (src[i + 2] << 8) | (src[i + 1] << 16) | (src[i] << 24)
133    }
134    offset = _writeCodepoint(dst, offset, codepoint, badChar)
135  }
136
137  // Keep overflowing bytes.
138  for (; i < src.length; i++) {
139    overflow.push(src[i])
140  }
141
142  return dst.slice(0, offset).toString("ucs2")
143}
144
145function _writeCodepoint (dst, offset, codepoint, badChar) {
146  // NOTE: codepoint is signed int32 and can be negative. We keep it that way to help V8 with optimizations.
147  if (codepoint < 0 || codepoint > 0x10FFFF) {
148    // Not a valid Unicode codepoint
149    codepoint = badChar
150  }
151
152  // Ephemeral Planes: Write high surrogate.
153  if (codepoint >= 0x10000) {
154    codepoint -= 0x10000
155
156    var high = 0xD800 | (codepoint >> 10)
157    dst[offset++] = high & 0xff
158    dst[offset++] = high >> 8
159
160    // Low surrogate is written below.
161    var codepoint = 0xDC00 | (codepoint & 0x3FF)
162  }
163
164  // Write BMP char or low surrogate.
165  dst[offset++] = codepoint & 0xff
166  dst[offset++] = codepoint >> 8
167
168  return offset
169};
170
171Utf32Decoder.prototype.end = function () {
172  this.overflow.length = 0
173}
174
175// == UTF-32 Auto codec =============================================================
176// Decoder chooses automatically from UTF-32LE and UTF-32BE using BOM and space-based heuristic.
177// Defaults to UTF-32LE. http://en.wikipedia.org/wiki/UTF-32
178// Encoder/decoder default can be changed: iconv.decode(buf, 'utf32', {defaultEncoding: 'utf-32be'});
179
180// Encoder prepends BOM (which can be overridden with (addBOM: false}).
181
182exports.utf32 = Utf32AutoCodec
183exports.ucs4 = "utf32"
184
185function Utf32AutoCodec (options, iconv) {
186  this.iconv = iconv
187}
188
189Utf32AutoCodec.prototype.encoder = Utf32AutoEncoder
190Utf32AutoCodec.prototype.decoder = Utf32AutoDecoder
191
192// -- Encoding
193
194function Utf32AutoEncoder (options, codec) {
195  options = options || {}
196
197  if (options.addBOM === undefined) {
198    options.addBOM = true
199  }
200
201  this.encoder = codec.iconv.getEncoder(options.defaultEncoding || "utf-32le", options)
202}
203
204Utf32AutoEncoder.prototype.write = function (str) {
205  return this.encoder.write(str)
206}
207
208Utf32AutoEncoder.prototype.end = function () {
209  return this.encoder.end()
210}
211
212// -- Decoding
213
214function Utf32AutoDecoder (options, codec) {
215  this.decoder = null
216  this.initialBufs = []
217  this.initialBufsLen = 0
218  this.options = options || {}
219  this.iconv = codec.iconv
220}
221
222Utf32AutoDecoder.prototype.write = function (buf) {
223  if (!this.decoder) {
224    // Codec is not chosen yet. Accumulate initial bytes.
225    this.initialBufs.push(buf)
226    this.initialBufsLen += buf.length
227
228    if (this.initialBufsLen < 32) // We need more bytes to use space heuristic (see below)
229    { return "" }
230
231    // We have enough bytes -> detect endianness.
232    var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding)
233    this.decoder = this.iconv.getDecoder(encoding, this.options)
234
235    var resStr = ""
236    for (var i = 0; i < this.initialBufs.length; i++) { resStr += this.decoder.write(this.initialBufs[i]) }
237
238    this.initialBufs.length = this.initialBufsLen = 0
239    return resStr
240  }
241
242  return this.decoder.write(buf)
243}
244
245Utf32AutoDecoder.prototype.end = function () {
246  if (!this.decoder) {
247    var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding)
248    this.decoder = this.iconv.getDecoder(encoding, this.options)
249
250    var resStr = ""
251    for (var i = 0; i < this.initialBufs.length; i++) { resStr += this.decoder.write(this.initialBufs[i]) }
252
253    var trail = this.decoder.end()
254    if (trail) { resStr += trail }
255
256    this.initialBufs.length = this.initialBufsLen = 0
257    return resStr
258  }
259
260  return this.decoder.end()
261}
262
263function detectEncoding (bufs, defaultEncoding) {
264  var b = []
265  var charsProcessed = 0
266  var invalidLE = 0; var invalidBE = 0   // Number of invalid chars when decoded as LE or BE.
267  var bmpCharsLE = 0; var bmpCharsBE = 0 // Number of BMP chars when decoded as LE or BE.
268
269  outerLoop:
270  for (var i = 0; i < bufs.length; i++) {
271    var buf = bufs[i]
272    for (var j = 0; j < buf.length; j++) {
273      b.push(buf[j])
274      if (b.length === 4) {
275        if (charsProcessed === 0) {
276          // Check BOM first.
277          if (b[0] === 0xFF && b[1] === 0xFE && b[2] === 0 && b[3] === 0) {
278            return "utf-32le"
279          }
280          if (b[0] === 0 && b[1] === 0 && b[2] === 0xFE && b[3] === 0xFF) {
281            return "utf-32be"
282          }
283        }
284
285        if (b[0] !== 0 || b[1] > 0x10) invalidBE++
286        if (b[3] !== 0 || b[2] > 0x10) invalidLE++
287
288        if (b[0] === 0 && b[1] === 0 && (b[2] !== 0 || b[3] !== 0)) bmpCharsBE++
289        if ((b[0] !== 0 || b[1] !== 0) && b[2] === 0 && b[3] === 0) bmpCharsLE++
290
291        b.length = 0
292        charsProcessed++
293
294        if (charsProcessed >= 100) {
295          break outerLoop
296        }
297      }
298    }
299  }
300
301  // Make decisions.
302  if (bmpCharsBE - invalidBE > bmpCharsLE - invalidLE) return "utf-32be"
303  if (bmpCharsBE - invalidBE < bmpCharsLE - invalidLE) return "utf-32le"
304
305  // Couldn't decide (likely all zeros or not enough data).
306  return defaultEncoding || "utf-32le"
307}
308