codekingpro/portable-devtools

1"use strict"
2
3// Description of supported double byte encodings and aliases.
4// Tables are not require()-d until they are needed to speed up library load.
5// require()-s are direct to support Browserify.
6
7module.exports = {
8
9  // == Japanese/ShiftJIS ====================================================
10  // All japanese encodings are based on JIS X set of standards:
11  // JIS X 0201 - Single-byte encoding of ASCII + ¥ + Kana chars at 0xA1-0xDF.
12  // JIS X 0208 - Main set of 6879 characters, placed in 94x94 plane, to be encoded by 2 bytes.
13  //              Has several variations in 1978, 1983, 1990 and 1997.
14  // JIS X 0212 - Supplementary plane of 6067 chars in 94x94 plane. 1990. Effectively dead.
15  // JIS X 0213 - Extension and modern replacement of 0208 and 0212. Total chars: 11233.
16  //              2 planes, first is superset of 0208, second - revised 0212.
17  //              Introduced in 2000, revised 2004. Some characters are in Unicode Plane 2 (0x2xxxx)
18
19  // Byte encodings are:
20  //  * Shift_JIS: Compatible with 0201, uses not defined chars in top half as lead bytes for double-byte
21  //               encoding of 0208. Lead byte ranges: 0x81-0x9F, 0xE0-0xEF; Trail byte ranges: 0x40-0x7E, 0x80-0x9E, 0x9F-0xFC.
22  //               Windows CP932 is a superset of Shift_JIS. Some companies added more chars, notably KDDI.
23  //  * EUC-JP:    Up to 3 bytes per character. Used mostly on *nixes.
24  //               0x00-0x7F       - lower part of 0201
25  //               0x8E, 0xA1-0xDF - upper part of 0201
26  //               (0xA1-0xFE)x2   - 0208 plane (94x94).
27  //               0x8F, (0xA1-0xFE)x2 - 0212 plane (94x94).
28  //  * JIS X 208: 7-bit, direct encoding of 0208. Byte ranges: 0x21-0x7E (94 values). Uncommon.
29  //               Used as-is in ISO2022 family.
30  //  * ISO2022-JP: Stateful encoding, with escape sequences to switch between ASCII,
31  //                0201-1976 Roman, 0208-1978, 0208-1983.
32  //  * ISO2022-JP-1: Adds esc seq for 0212-1990.
33  //  * ISO2022-JP-2: Adds esc seq for GB2313-1980, KSX1001-1992, ISO8859-1, ISO8859-7.
34  //  * ISO2022-JP-3: Adds esc seq for 0201-1976 Kana set, 0213-2000 Planes 1, 2.
35  //  * ISO2022-JP-2004: Adds 0213-2004 Plane 1.
36  //
37  // After JIS X 0213 appeared, Shift_JIS-2004, EUC-JISX0213 and ISO2022-JP-2004 followed, with just changing the planes.
38  //
39  // Overall, it seems that it's a mess :( http://www8.plala.or.jp/tkubota1/unicode-symbols-map2.html
40
41  shiftjis: {
42    type: "_dbcs",
43    table: function () { return require("./tables/shiftjis.json") },
44    encodeAdd: { "\u00a5": 0x5C, "\u203E": 0x7E },
45    encodeSkipVals: [{ from: 0xED40, to: 0xF940 }]
46  },
47  csshiftjis: "shiftjis",
48  mskanji: "shiftjis",
49  sjis: "shiftjis",
50  windows31j: "shiftjis",
51  ms31j: "shiftjis",
52  xsjis: "shiftjis",
53  windows932: "shiftjis",
54  ms932: "shiftjis",
55  932: "shiftjis",
56  cp932: "shiftjis",
57
58  eucjp: {
59    type: "_dbcs",
60    table: function () { return require("./tables/eucjp.json") },
61    encodeAdd: { "\u00a5": 0x5C, "\u203E": 0x7E }
62  },
63
64  // TODO: KDDI extension to Shift_JIS
65  // TODO: IBM CCSID 942 = CP932, but F0-F9 custom chars and other char changes.
66  // TODO: IBM CCSID 943 = Shift_JIS = CP932 with original Shift_JIS lower 128 chars.
67
68  // == Chinese/GBK ==========================================================
69  // http://en.wikipedia.org/wiki/GBK
70  // We mostly implement W3C recommendation: https://www.w3.org/TR/encoding/#gbk-encoder
71
72  // Oldest GB2312 (1981, ~7600 chars) is a subset of CP936
73  gb2312: "cp936",
74  gb231280: "cp936",
75  gb23121980: "cp936",
76  csgb2312: "cp936",
77  csiso58gb231280: "cp936",
78  euccn: "cp936",
79
80  // Microsoft's CP936 is a subset and approximation of GBK.
81  windows936: "cp936",
82  ms936: "cp936",
83  936: "cp936",
84  cp936: {
85    type: "_dbcs",
86    table: function () { return require("./tables/cp936.json") }
87  },
88
89  // GBK (~22000 chars) is an extension of CP936 that added user-mapped chars and some other.
90  gbk: {
91    type: "_dbcs",
92    table: function () { return require("./tables/cp936.json").concat(require("./tables/gbk-added.json")) }
93  },
94  xgbk: "gbk",
95  isoir58: "gbk",
96
97  // GB18030 is an algorithmic extension of GBK.
98  // Main source: https://www.w3.org/TR/encoding/#gbk-encoder
99  // http://icu-project.org/docs/papers/gb18030.html
100  // http://source.icu-project.org/repos/icu/data/trunk/charset/data/xml/gb-18030-2000.xml
101  // http://www.khngai.com/chinese/charmap/tblgbk.php?page=0
102  gb18030: {
103    type: "_dbcs",
104    table: function () { return require("./tables/cp936.json").concat(require("./tables/gbk-added.json")) },
105    gb18030: function () { return require("./tables/gb18030-ranges.json") },
106    encodeSkipVals: [0x80],
107    encodeAdd: { "€": 0xA2E3 }
108  },
109
110  chinese: "gb18030",
111
112  // == Korean ===============================================================
113  // EUC-KR, KS_C_5601 and KS X 1001 are exactly the same.
114  windows949: "cp949",
115  ms949: "cp949",
116  949: "cp949",
117  cp949: {
118    type: "_dbcs",
119    table: function () { return require("./tables/cp949.json") }
120  },
121
122  cseuckr: "cp949",
123  csksc56011987: "cp949",
124  euckr: "cp949",
125  isoir149: "cp949",
126  korean: "cp949",
127  ksc56011987: "cp949",
128  ksc56011989: "cp949",
129  ksc5601: "cp949",
130
131  // == Big5/Taiwan/Hong Kong ================================================
132  // There are lots of tables for Big5 and cp950. Please see the following links for history:
133  // http://moztw.org/docs/big5/  http://www.haible.de/bruno/charsets/conversion-tables/Big5.html
134  // Variations, in roughly number of defined chars:
135  //  * Windows CP 950: Microsoft variant of Big5. Canonical: http://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP950.TXT
136  //  * Windows CP 951: Microsoft variant of Big5-HKSCS-2001. Seems to be never public. http://me.abelcheung.org/articles/research/what-is-cp951/
137  //  * Big5-2003 (Taiwan standard) almost superset of cp950.
138  //  * Unicode-at-on (UAO) / Mozilla 1.8. Falling out of use on the Web. Not supported by other browsers.
139  //  * Big5-HKSCS (-2001, -2004, -2008). Hong Kong standard.
140  //    many unicode code points moved from PUA to Supplementary plane (U+2XXXX) over the years.
141  //    Plus, it has 4 combining sequences.
142  //    Seems that Mozilla refused to support it for 10 yrs. https://bugzilla.mozilla.org/show_bug.cgi?id=162431 https://bugzilla.mozilla.org/show_bug.cgi?id=310299
143  //    because big5-hkscs is the only encoding to include astral characters in non-algorithmic way.
144  //    Implementations are not consistent within browsers; sometimes labeled as just big5.
145  //    MS Internet Explorer switches from big5 to big5-hkscs when a patch applied.
146  //    Great discussion & recap of what's going on https://bugzilla.mozilla.org/show_bug.cgi?id=912470#c31
147  //    In the encoder, it might make sense to support encoding old PUA mappings to Big5 bytes seq-s.
148  //    Official spec: http://www.ogcio.gov.hk/en/business/tech_promotion/ccli/terms/doc/2003cmp_2008.txt
149  //                   http://www.ogcio.gov.hk/tc/business/tech_promotion/ccli/terms/doc/hkscs-2008-big5-iso.txt
150  //
151  // Current understanding of how to deal with Big5(-HKSCS) is in the Encoding Standard, http://encoding.spec.whatwg.org/#big5-encoder
152  // Unicode mapping (http://www.unicode.org/Public/MAPPINGS/OBSOLETE/EASTASIA/OTHER/BIG5.TXT) is said to be wrong.
153
154  windows950: "cp950",
155  ms950: "cp950",
156  950: "cp950",
157  cp950: {
158    type: "_dbcs",
159    table: function () { return require("./tables/cp950.json") }
160  },
161
162  // Big5 has many variations and is an extension of cp950. We use Encoding Standard's as a consensus.
163  big5: "big5hkscs",
164  big5hkscs: {
165    type: "_dbcs",
166    table: function () { return require("./tables/cp950.json").concat(require("./tables/big5-added.json")) },
167    encodeSkipVals: [
168      // Although Encoding Standard says we should avoid encoding to HKSCS area (See Step 1 of
169      // https://encoding.spec.whatwg.org/#index-big5-pointer), we still do it to increase compatibility with ICU.
170      // But if a single unicode point can be encoded both as HKSCS and regular Big5, we prefer the latter.
171      0x8e69, 0x8e6f, 0x8e7e, 0x8eab, 0x8eb4, 0x8ecd, 0x8ed0, 0x8f57, 0x8f69, 0x8f6e, 0x8fcb, 0x8ffe,
172      0x906d, 0x907a, 0x90c4, 0x90dc, 0x90f1, 0x91bf, 0x92af, 0x92b0, 0x92b1, 0x92b2, 0x92d1, 0x9447, 0x94ca,
173      0x95d9, 0x96fc, 0x9975, 0x9b76, 0x9b78, 0x9b7b, 0x9bc6, 0x9bde, 0x9bec, 0x9bf6, 0x9c42, 0x9c53, 0x9c62,
174      0x9c68, 0x9c6b, 0x9c77, 0x9cbc, 0x9cbd, 0x9cd0, 0x9d57, 0x9d5a, 0x9dc4, 0x9def, 0x9dfb, 0x9ea9, 0x9eef,
175      0x9efd, 0x9f60, 0x9fcb, 0xa077, 0xa0dc, 0xa0df, 0x8fcc, 0x92c8, 0x9644, 0x96ed,
176
177      // Step 2 of https://encoding.spec.whatwg.org/#index-big5-pointer: Use last pointer for U+2550, U+255E, U+2561, U+256A, U+5341, or U+5345
178      0xa2a4, 0xa2a5, 0xa2a7, 0xa2a6, 0xa2cc, 0xa2ce
179    ]
180  },
181
182  cnbig5: "big5hkscs",
183  csbig5: "big5hkscs",
184  xxbig5: "big5hkscs"
185}
186