Brunobkr/llama.cpp_AlgMor24_github

1/*2 * This file defines the Unicode scripts and script families that we3 * support. To add new scripts or families, just add a new entry to the4 * scriptData array below. Adding scripts to the scriptData array allows5 * characters from that script to appear in \text{} environments.6 */7 8/**9 * Each script or script family has a name and an array of blocks.10 * Each block is an array of two numbers which specify the start and11 * end points (inclusive) of a block of Unicode codepoints.12 */13type Script = {14    name: string;15    blocks: Array<Array<number>>;16};17 18/**19 * Unicode block data for the families of scripts we support in \text{}.20 * Scripts only need to appear here if they do not have font metrics.21 */22const scriptData: Array<Script> = [23    {24        // Latin characters beyond the Latin-1 characters we have metrics for.25        // Needed for Czech, Hungarian and Turkish text, for example.26        name: 'latin',27        blocks: [28            [0x0100, 0x024f],  // Latin Extended-A and Latin Extended-B29            [0x0300, 0x036f],  // Combining Diacritical marks30        ],31    },32    {33        // The Cyrillic script used by Russian and related languages.34        // A Cyrillic subset used to be supported as explicitly defined35        // symbols in symbols.js36        name: 'cyrillic',37        blocks: [[0x0400, 0x04ff]],38    },39    {40        // Armenian41        name: 'armenian',42        blocks: [[0x0530, 0x058F]],43    },44    {45        // The Brahmic scripts of South and Southeast Asia46        // Devanagari (0900–097F)47        // Bengali (0980–09FF)48        // Gurmukhi (0A00–0A7F)49        // Gujarati (0A80–0AFF)50        // Oriya (0B00–0B7F)51        // Tamil (0B80–0BFF)52        // Telugu (0C00–0C7F)53        // Kannada (0C80–0CFF)54        // Malayalam (0D00–0D7F)55        // Sinhala (0D80–0DFF)56        // Thai (0E00–0E7F)57        // Lao (0E80–0EFF)58        // Tibetan (0F00–0FFF)59        // Myanmar (1000–109F)60        name: 'brahmic',61        blocks: [[0x0900, 0x109F]],62    },63    {64        name: 'georgian',65        blocks: [[0x10A0, 0x10ff]],66    },67    {68        // Chinese and Japanese.69        // The "k" in cjk is for Korean, but we've separated Korean out70        name: "cjk",71        blocks: [72            [0x3000, 0x30FF], // CJK symbols and punctuation, Hiragana, Katakana73            [0x4E00, 0x9FAF], // CJK ideograms74            [0xFF00, 0xFF60], // Fullwidth punctuation75            // TODO: add halfwidth Katakana and Romanji glyphs76        ],77    },78    {79        // Korean80        name: 'hangul',81        blocks: [[0xAC00, 0xD7AF]],82    },83];84 85/**86 * Given a codepoint, return the name of the script or script family87 * it is from, or null if it is not part of a known block88 */89export function scriptFromCodepoint(codepoint: number): string | null | undefined {90    for (let i = 0; i < scriptData.length; i++) {91        const script = scriptData[i];92        for (let i = 0; i < script.blocks.length; i++) {93            const block = script.blocks[i];94            if (codepoint >= block[0] && codepoint <= block[1]) {95                return script.name;96            }97        }98    }99    return null;100}101 102/**103 * A flattened version of all the supported blocks in a single array.104 * This is an optimization to make supportedCodepoint() fast.105 */106const allBlocks: Array<number> = [];107scriptData.forEach(s => s.blocks.forEach(b => allBlocks.push(...b)));108 109/**110 * Given a codepoint, return true if it falls within one of the111 * scripts or script families defined above and false otherwise.112 *113 * Micro benchmarks shows that this is faster than114 * /[\u3000-\u30FF\u4E00-\u9FAF\uFF00-\uFF60\uAC00-\uD7AF\u0900-\u109F]/.test()115 * in Firefox, Chrome and Node.116 */117export function supportedCodepoint(codepoint: number): boolean {118    for (let i = 0; i < allBlocks.length; i += 2) {119        if (codepoint >= allBlocks[i] && codepoint <= allBlocks[i + 1]) {120            return true;121        }122    }123    return false;124}125