codekingpro/portable-devtools
114k
1"use strict"
2var Buffer = require("safer-buffer").Buffer
3
4// Multibyte codec. In this scheme, a character is represented by 1 or more bytes.
5// Our codec supports UTF-16 surrogates, extensions for GB18030 and unicode sequences.
6// To save memory and loading time, we read table files only when requested.
7
8exports._dbcs = DBCSCodec
9
10var UNASSIGNED = -1
11var GB18030_CODE = -2
12var SEQ_START = -10
13var NODE_START = -1000
14var UNASSIGNED_NODE = new Array(0x100)
15var DEF_CHAR = -1
16
17for (var i = 0; i < 0x100; i++) { UNASSIGNED_NODE[i] = UNASSIGNED }
18
19// Class DBCSCodec reads and initializes mapping tables.
20function DBCSCodec (codecOptions, iconv) {
21 this.encodingName = codecOptions.encodingName
22 if (!codecOptions) { throw new Error("DBCS codec is called without the data.") }
23 if (!codecOptions.table) { throw new Error("Encoding '" + this.encodingName + "' has no data.") }
24
25 // Load tables.
26 var mappingTable = codecOptions.table()
27
28 // Decode tables: MBCS -> Unicode.
29
30 // decodeTables is a trie, encoded as an array of arrays of integers. Internal arrays are trie nodes and all have len = 256.
31 // Trie root is decodeTables[0].
32 // Values: >= 0 -> unicode character code. can be > 0xFFFF
33 // == UNASSIGNED -> unknown/unassigned sequence.
34 // == GB18030_CODE -> this is the end of a GB18030 4-byte sequence.
35 // <= NODE_START -> index of the next node in our trie to process next byte.
36 // <= SEQ_START -> index of the start of a character code sequence, in decodeTableSeq.
37 this.decodeTables = []
38 this.decodeTables[0] = UNASSIGNED_NODE.slice(0) // Create root node.
39
40 // Sometimes a MBCS char corresponds to a sequence of unicode chars. We store them as arrays of integers here.
41 this.decodeTableSeq = []
42
43 // Actual mapping tables consist of chunks. Use them to fill up decode tables.
44 for (var i = 0; i < mappingTable.length; i++) { this._addDecodeChunk(mappingTable[i]) }
45
46 // Load & create GB18030 tables when needed.
47 if (typeof codecOptions.gb18030 === "function") {
48 this.gb18030 = codecOptions.gb18030() // Load GB18030 ranges.
49
50 // Add GB18030 common decode nodes.
51 var commonThirdByteNodeIdx = this.decodeTables.length
52 this.decodeTables.push(UNASSIGNED_NODE.slice(0))
53
54 var commonFourthByteNodeIdx = this.decodeTables.length
55 this.decodeTables.push(UNASSIGNED_NODE.slice(0))
56
57 // Fill out the tree
58 var firstByteNode = this.decodeTables[0]
59 for (var i = 0x81; i <= 0xFE; i++) {
60 var secondByteNode = this.decodeTables[NODE_START - firstByteNode[i]]
61 for (var j = 0x30; j <= 0x39; j++) {
62 if (secondByteNode[j] === UNASSIGNED) {
63 secondByteNode[j] = NODE_START - commonThirdByteNodeIdx
64 } else if (secondByteNode[j] > NODE_START) {
65 throw new Error("gb18030 decode tables conflict at byte 2")
66 }
67
68 var thirdByteNode = this.decodeTables[NODE_START - secondByteNode[j]]
69 for (var k = 0x81; k <= 0xFE; k++) {
70 if (thirdByteNode[k] === UNASSIGNED) {
71 thirdByteNode[k] = NODE_START - commonFourthByteNodeIdx
72 } else if (thirdByteNode[k] === NODE_START - commonFourthByteNodeIdx) {
73 continue
74 } else if (thirdByteNode[k] > NODE_START) {
75 throw new Error("gb18030 decode tables conflict at byte 3")
76 }
77
78 var fourthByteNode = this.decodeTables[NODE_START - thirdByteNode[k]]
79 for (var l = 0x30; l <= 0x39; l++) {
80 if (fourthByteNode[l] === UNASSIGNED) { fourthByteNode[l] = GB18030_CODE }
81 }
82 }
83 }
84 }
85 }
86
87 this.defaultCharUnicode = iconv.defaultCharUnicode
88
89 // Encode tables: Unicode -> DBCS.
90
91 // `encodeTable` is array mapping from unicode char to encoded char. All its values are integers for performance.
92 // Because it can be sparse, it is represented as array of buckets by 256 chars each. Bucket can be null.
93 // Values: >= 0 -> it is a normal char. Write the value (if <=256 then 1 byte, if <=65536 then 2 bytes, etc.).
94 // == UNASSIGNED -> no conversion found. Output a default char.
95 // <= SEQ_START -> it's an index in encodeTableSeq, see below. The character starts a sequence.
96 this.encodeTable = []
97
98 // `encodeTableSeq` is used when a sequence of unicode characters is encoded as a single code. We use a tree of
99 // objects where keys correspond to characters in sequence and leafs are the encoded dbcs values. A special DEF_CHAR key
100 // means end of sequence (needed when one sequence is a strict subsequence of another).
101 // Objects are kept separately from encodeTable to increase performance.
102 this.encodeTableSeq = []
103
104 // Some chars can be decoded, but need not be encoded.
105 var skipEncodeChars = {}
106 if (codecOptions.encodeSkipVals) {
107 for (var i = 0; i < codecOptions.encodeSkipVals.length; i++) {
108 var val = codecOptions.encodeSkipVals[i]
109 if (typeof val === "number") { skipEncodeChars[val] = true } else {
110 for (var j = val.from; j <= val.to; j++) { skipEncodeChars[j] = true }
111 }
112 }
113 }
114
115 // Use decode trie to recursively fill out encode tables.
116 this._fillEncodeTable(0, 0, skipEncodeChars)
117
118 // Add more encoding pairs when needed.
119 if (codecOptions.encodeAdd) {
120 for (var uChar in codecOptions.encodeAdd) {
121 if (Object.prototype.hasOwnProperty.call(codecOptions.encodeAdd, uChar)) { this._setEncodeChar(uChar.charCodeAt(0), codecOptions.encodeAdd[uChar]) }
122 }
123 }
124
125 this.defCharSB = this.encodeTable[0][iconv.defaultCharSingleByte.charCodeAt(0)]
126 if (this.defCharSB === UNASSIGNED) this.defCharSB = this.encodeTable[0]["?"]
127 if (this.defCharSB === UNASSIGNED) this.defCharSB = "?".charCodeAt(0)
128}
129
130DBCSCodec.prototype.encoder = DBCSEncoder
131DBCSCodec.prototype.decoder = DBCSDecoder
132
133// Decoder helpers
134DBCSCodec.prototype._getDecodeTrieNode = function (addr) {
135 var bytes = []
136 for (; addr > 0; addr >>>= 8) { bytes.push(addr & 0xFF) }
137 if (bytes.length == 0) { bytes.push(0) }
138
139 var node = this.decodeTables[0]
140 for (var i = bytes.length - 1; i > 0; i--) { // Traverse nodes deeper into the trie.
141 var val = node[bytes[i]]
142
143 if (val == UNASSIGNED) { // Create new node.
144 node[bytes[i]] = NODE_START - this.decodeTables.length
145 this.decodeTables.push(node = UNASSIGNED_NODE.slice(0))
146 } else if (val <= NODE_START) { // Existing node.
147 node = this.decodeTables[NODE_START - val]
148 } else { throw new Error("Overwrite byte in " + this.encodingName + ", addr: " + addr.toString(16)) }
149 }
150 return node
151}
152
153DBCSCodec.prototype._addDecodeChunk = function (chunk) {
154 // First element of chunk is the hex mbcs code where we start.
155 var curAddr = parseInt(chunk[0], 16)
156
157 // Choose the decoding node where we'll write our chars.
158 var writeTable = this._getDecodeTrieNode(curAddr)
159 curAddr = curAddr & 0xFF
160
161 // Write all other elements of the chunk to the table.
162 for (var k = 1; k < chunk.length; k++) {
163 var part = chunk[k]
164 if (typeof part === "string") { // String, write as-is.
165 for (var l = 0; l < part.length;) {
166 var code = part.charCodeAt(l++)
167 if (code >= 0xD800 && code < 0xDC00) { // Decode surrogate
168 var codeTrail = part.charCodeAt(l++)
169 if (codeTrail >= 0xDC00 && codeTrail < 0xE000) { writeTable[curAddr++] = 0x10000 + (code - 0xD800) * 0x400 + (codeTrail - 0xDC00) } else { throw new Error("Incorrect surrogate pair in " + this.encodingName + " at chunk " + chunk[0]) }
170 } else if (code > 0x0FF0 && code <= 0x0FFF) { // Character sequence (our own encoding used)
171 var len = 0xFFF - code + 2
172 var seq = []
173 for (var m = 0; m < len; m++) { seq.push(part.charCodeAt(l++)) } // Simple variation: don't support surrogates or subsequences in seq.
174
175 writeTable[curAddr++] = SEQ_START - this.decodeTableSeq.length
176 this.decodeTableSeq.push(seq)
177 } else { writeTable[curAddr++] = code } // Basic char
178 }
179 } else if (typeof part === "number") { // Integer, meaning increasing sequence starting with prev character.
180 var charCode = writeTable[curAddr - 1] + 1
181 for (var l = 0; l < part; l++) { writeTable[curAddr++] = charCode++ }
182 } else { throw new Error("Incorrect type '" + typeof part + "' given in " + this.encodingName + " at chunk " + chunk[0]) }
183 }
184 if (curAddr > 0xFF) { throw new Error("Incorrect chunk in " + this.encodingName + " at addr " + chunk[0] + ": too long" + curAddr) }
185}
186
187// Encoder helpers
188DBCSCodec.prototype._getEncodeBucket = function (uCode) {
189 var high = uCode >> 8 // This could be > 0xFF because of astral characters.
190 if (this.encodeTable[high] === undefined) {
191 this.encodeTable[high] = UNASSIGNED_NODE.slice(0)
192 } // Create bucket on demand.
193 return this.encodeTable[high]
194}
195
196DBCSCodec.prototype._setEncodeChar = function (uCode, dbcsCode) {
197 var bucket = this._getEncodeBucket(uCode)
198 var low = uCode & 0xFF
199 if (bucket[low] <= SEQ_START) { this.encodeTableSeq[SEQ_START - bucket[low]][DEF_CHAR] = dbcsCode } // There's already a sequence, set a single-char subsequence of it.
200 else if (bucket[low] == UNASSIGNED) { bucket[low] = dbcsCode }
201}
202
203DBCSCodec.prototype._setEncodeSequence = function (seq, dbcsCode) {
204 // Get the root of character tree according to first character of the sequence.
205 var uCode = seq[0]
206 var bucket = this._getEncodeBucket(uCode)
207 var low = uCode & 0xFF
208
209 var node
210 if (bucket[low] <= SEQ_START) {
211 // There's already a sequence with - use it.
212 node = this.encodeTableSeq[SEQ_START - bucket[low]]
213 } else {
214 // There was no sequence object - allocate a new one.
215 node = {}
216 if (bucket[low] !== UNASSIGNED) node[DEF_CHAR] = bucket[low] // If a char was set before - make it a single-char subsequence.
217 bucket[low] = SEQ_START - this.encodeTableSeq.length
218 this.encodeTableSeq.push(node)
219 }
220
221 // Traverse the character tree, allocating new nodes as needed.
222 for (var j = 1; j < seq.length - 1; j++) {
223 var oldVal = node[uCode]
224 if (typeof oldVal === "object") { node = oldVal } else {
225 node = node[uCode] = {}
226 if (oldVal !== undefined) { node[DEF_CHAR] = oldVal }
227 }
228 }
229
230 // Set the leaf to given dbcsCode.
231 uCode = seq[seq.length - 1]
232 node[uCode] = dbcsCode
233}
234
235DBCSCodec.prototype._fillEncodeTable = function (nodeIdx, prefix, skipEncodeChars) {
236 var node = this.decodeTables[nodeIdx]
237 var hasValues = false
238 var subNodeEmpty = {}
239 for (var i = 0; i < 0x100; i++) {
240 var uCode = node[i]
241 var mbCode = prefix + i
242 if (skipEncodeChars[mbCode]) { continue }
243
244 if (uCode >= 0) {
245 this._setEncodeChar(uCode, mbCode)
246 hasValues = true
247 } else if (uCode <= NODE_START) {
248 var subNodeIdx = NODE_START - uCode
249 if (!subNodeEmpty[subNodeIdx]) { // Skip empty subtrees (they are too large in gb18030).
250 var newPrefix = (mbCode << 8) >>> 0 // NOTE: '>>> 0' keeps 32-bit num positive.
251 if (this._fillEncodeTable(subNodeIdx, newPrefix, skipEncodeChars)) { hasValues = true } else { subNodeEmpty[subNodeIdx] = true }
252 }
253 } else if (uCode <= SEQ_START) {
254 this._setEncodeSequence(this.decodeTableSeq[SEQ_START - uCode], mbCode)
255 hasValues = true
256 }
257 }
258 return hasValues
259}
260
261// == Encoder ==================================================================
262
263function DBCSEncoder (options, codec) {
264 // Encoder state
265 this.leadSurrogate = -1
266 this.seqObj = undefined
267
268 // Static data
269 this.encodeTable = codec.encodeTable
270 this.encodeTableSeq = codec.encodeTableSeq
271 this.defaultCharSingleByte = codec.defCharSB
272 this.gb18030 = codec.gb18030
273}
274
275DBCSEncoder.prototype.write = function (str) {
276 var newBuf = Buffer.alloc(str.length * (this.gb18030 ? 4 : 3))
277 var leadSurrogate = this.leadSurrogate
278 var seqObj = this.seqObj
279 var nextChar = -1
280 var i = 0; var j = 0
281
282 while (true) {
283 // 0. Get next character.
284 if (nextChar === -1) {
285 if (i == str.length) break
286 var uCode = str.charCodeAt(i++)
287 } else {
288 var uCode = nextChar
289 nextChar = -1
290 }
291
292 // 1. Handle surrogates.
293 if (uCode >= 0xD800 && uCode < 0xE000) { // Char is one of surrogates.
294 if (uCode < 0xDC00) { // We've got lead surrogate.
295 if (leadSurrogate === -1) {
296 leadSurrogate = uCode
297 continue
298 } else {
299 leadSurrogate = uCode
300 // Double lead surrogate found.
301 uCode = UNASSIGNED
302 }
303 } else { // We've got trail surrogate.
304 if (leadSurrogate !== -1) {
305 uCode = 0x10000 + (leadSurrogate - 0xD800) * 0x400 + (uCode - 0xDC00)
306 leadSurrogate = -1
307 } else {
308 // Incomplete surrogate pair - only trail surrogate found.
309 uCode = UNASSIGNED
310 }
311 }
312 } else if (leadSurrogate !== -1) {
313 // Incomplete surrogate pair - only lead surrogate found.
314 nextChar = uCode; uCode = UNASSIGNED // Write an error, then current char.
315 leadSurrogate = -1
316 }
317
318 // 2. Convert uCode character.
319 var dbcsCode = UNASSIGNED
320 if (seqObj !== undefined && uCode != UNASSIGNED) { // We are in the middle of the sequence
321 var resCode = seqObj[uCode]
322 if (typeof resCode === "object") { // Sequence continues.
323 seqObj = resCode
324 continue
325 } else if (typeof resCode === "number") { // Sequence finished. Write it.
326 dbcsCode = resCode
327 } else if (resCode == undefined) { // Current character is not part of the sequence.
328 // Try default character for this sequence
329 resCode = seqObj[DEF_CHAR]
330 if (resCode !== undefined) {
331 dbcsCode = resCode // Found. Write it.
332 nextChar = uCode // Current character will be written too in the next iteration.
333 } else {
334 // TODO: What if we have no default? (resCode == undefined)
335 // Then, we should write first char of the sequence as-is and try the rest recursively.
336 // Didn't do it for now because no encoding has this situation yet.
337 // Currently, just skip the sequence and write current char.
338 }
339 }
340 seqObj = undefined
341 } else if (uCode >= 0) { // Regular character
342 var subtable = this.encodeTable[uCode >> 8]
343 if (subtable !== undefined) { dbcsCode = subtable[uCode & 0xFF] }
344
345 if (dbcsCode <= SEQ_START) { // Sequence start
346 seqObj = this.encodeTableSeq[SEQ_START - dbcsCode]
347 continue
348 }
349
350 if (dbcsCode == UNASSIGNED && this.gb18030) {
351 // Use GB18030 algorithm to find character(s) to write.
352 var idx = findIdx(this.gb18030.uChars, uCode)
353 if (idx != -1) {
354 var dbcsCode = this.gb18030.gbChars[idx] + (uCode - this.gb18030.uChars[idx])
355 newBuf[j++] = 0x81 + Math.floor(dbcsCode / 12600); dbcsCode = dbcsCode % 12600
356 newBuf[j++] = 0x30 + Math.floor(dbcsCode / 1260); dbcsCode = dbcsCode % 1260
357 newBuf[j++] = 0x81 + Math.floor(dbcsCode / 10); dbcsCode = dbcsCode % 10
358 newBuf[j++] = 0x30 + dbcsCode
359 continue
360 }
361 }
362 }
363
364 // 3. Write dbcsCode character.
365 if (dbcsCode === UNASSIGNED) { dbcsCode = this.defaultCharSingleByte }
366
367 if (dbcsCode < 0x100) {
368 newBuf[j++] = dbcsCode
369 } else if (dbcsCode < 0x10000) {
370 newBuf[j++] = dbcsCode >> 8 // high byte
371 newBuf[j++] = dbcsCode & 0xFF // low byte
372 } else if (dbcsCode < 0x1000000) {
373 newBuf[j++] = dbcsCode >> 16
374 newBuf[j++] = (dbcsCode >> 8) & 0xFF
375 newBuf[j++] = dbcsCode & 0xFF
376 } else {
377 newBuf[j++] = dbcsCode >>> 24
378 newBuf[j++] = (dbcsCode >>> 16) & 0xFF
379 newBuf[j++] = (dbcsCode >>> 8) & 0xFF
380 newBuf[j++] = dbcsCode & 0xFF
381 }
382 }
383
384 this.seqObj = seqObj
385 this.leadSurrogate = leadSurrogate
386 return newBuf.slice(0, j)
387}
388
389DBCSEncoder.prototype.end = function () {
390 if (this.leadSurrogate === -1 && this.seqObj === undefined) { return } // All clean. Most often case.
391
392 var newBuf = Buffer.alloc(10); var j = 0
393
394 if (this.seqObj) { // We're in the sequence.
395 var dbcsCode = this.seqObj[DEF_CHAR]
396 if (dbcsCode !== undefined) { // Write beginning of the sequence.
397 if (dbcsCode < 0x100) {
398 newBuf[j++] = dbcsCode
399 } else {
400 newBuf[j++] = dbcsCode >> 8 // high byte
401 newBuf[j++] = dbcsCode & 0xFF // low byte
402 }
403 } else {
404 // See todo above.
405 }
406 this.seqObj = undefined
407 }
408
409 if (this.leadSurrogate !== -1) {
410 // Incomplete surrogate pair - only lead surrogate found.
411 newBuf[j++] = this.defaultCharSingleByte
412 this.leadSurrogate = -1
413 }
414
415 return newBuf.slice(0, j)
416}
417
418// Export for testing
419DBCSEncoder.prototype.findIdx = findIdx
420
421// == Decoder ==================================================================
422
423function DBCSDecoder (options, codec) {
424 // Decoder state
425 this.nodeIdx = 0
426 this.prevBytes = []
427
428 // Static data
429 this.decodeTables = codec.decodeTables
430 this.decodeTableSeq = codec.decodeTableSeq
431 this.defaultCharUnicode = codec.defaultCharUnicode
432 this.gb18030 = codec.gb18030
433}
434
435DBCSDecoder.prototype.write = function (buf) {
436 var newBuf = Buffer.alloc(buf.length * 2)
437 var nodeIdx = this.nodeIdx
438 var prevBytes = this.prevBytes; var prevOffset = this.prevBytes.length
439 var seqStart = -this.prevBytes.length // idx of the start of current parsed sequence.
440 var uCode
441
442 for (var i = 0, j = 0; i < buf.length; i++) {
443 var curByte = (i >= 0) ? buf[i] : prevBytes[i + prevOffset]
444
445 // Lookup in current trie node.
446 var uCode = this.decodeTables[nodeIdx][curByte]
447
448 if (uCode >= 0) {
449 // Normal character, just use it.
450 } else if (uCode === UNASSIGNED) { // Unknown char.
451 // TODO: Callback with seq.
452 uCode = this.defaultCharUnicode.charCodeAt(0)
453 i = seqStart // Skip one byte ('i' will be incremented by the for loop) and try to parse again.
454 } else if (uCode === GB18030_CODE) {
455 if (i >= 3) {
456 var ptr = (buf[i - 3] - 0x81) * 12600 + (buf[i - 2] - 0x30) * 1260 + (buf[i - 1] - 0x81) * 10 + (curByte - 0x30)
457 } else {
458 var ptr = (prevBytes[i - 3 + prevOffset] - 0x81) * 12600 +
459 (((i - 2 >= 0) ? buf[i - 2] : prevBytes[i - 2 + prevOffset]) - 0x30) * 1260 +
460 (((i - 1 >= 0) ? buf[i - 1] : prevBytes[i - 1 + prevOffset]) - 0x81) * 10 +
461 (curByte - 0x30)
462 }
463 var idx = findIdx(this.gb18030.gbChars, ptr)
464 uCode = this.gb18030.uChars[idx] + ptr - this.gb18030.gbChars[idx]
465 } else if (uCode <= NODE_START) { // Go to next trie node.
466 nodeIdx = NODE_START - uCode
467 continue
468 } else if (uCode <= SEQ_START) { // Output a sequence of chars.
469 var seq = this.decodeTableSeq[SEQ_START - uCode]
470 for (var k = 0; k < seq.length - 1; k++) {
471 uCode = seq[k]
472 newBuf[j++] = uCode & 0xFF
473 newBuf[j++] = uCode >> 8
474 }
475 uCode = seq[seq.length - 1]
476 } else { throw new Error("iconv-lite internal error: invalid decoding table value " + uCode + " at " + nodeIdx + "/" + curByte) }
477
478 // Write the character to buffer, handling higher planes using surrogate pair.
479 if (uCode >= 0x10000) {
480 uCode -= 0x10000
481 var uCodeLead = 0xD800 | (uCode >> 10)
482 newBuf[j++] = uCodeLead & 0xFF
483 newBuf[j++] = uCodeLead >> 8
484
485 uCode = 0xDC00 | (uCode & 0x3FF)
486 }
487 newBuf[j++] = uCode & 0xFF
488 newBuf[j++] = uCode >> 8
489
490 // Reset trie node.
491 nodeIdx = 0; seqStart = i + 1
492 }
493
494 this.nodeIdx = nodeIdx
495 this.prevBytes = (seqStart >= 0)
496 ? Array.prototype.slice.call(buf, seqStart)
497 : prevBytes.slice(seqStart + prevOffset).concat(Array.prototype.slice.call(buf))
498
499 return newBuf.slice(0, j).toString("ucs2")
500}
501
502DBCSDecoder.prototype.end = function () {
503 var ret = ""
504
505 // Try to parse all remaining chars.
506 while (this.prevBytes.length > 0) {
507 // Skip 1 character in the buffer.
508 ret += this.defaultCharUnicode
509 var bytesArr = this.prevBytes.slice(1)
510
511 // Parse remaining as usual.
512 this.prevBytes = []
513 this.nodeIdx = 0
514 if (bytesArr.length > 0) { ret += this.write(bytesArr) }
515 }
516
517 this.prevBytes = []
518 this.nodeIdx = 0
519 return ret
520}
521
522// Binary search for GB18030. Returns largest i such that table[i] <= val.
523function findIdx (table, val) {
524 if (table[0] > val) { return -1 }
525
526 var l = 0; var r = table.length
527 while (l < r - 1) { // always table[l] <= val < table[r]
528 var mid = l + ((r - l + 1) >> 1)
529 if (table[mid] <= val) { l = mid } else { r = mid }
530 }
531 return l
532}
533