Brunobkr/llama.cpp_AlgMor24_github

1import { htmlTrie } from "./generated/encode-html.js";2import { xmlReplacer, getCodePoint } from "./escape.js";3 4const htmlReplacer = /[\t\n\f!-,./:-@[-`{-}\u0080-\uFFFF]/g;5 6/**7 * Encodes all characters in the input using HTML entities. This includes8 * characters that are valid ASCII characters in HTML documents, such as `#`.9 *10 * To get a more compact output, consider using the `encodeNonAsciiHTML`11 * function, which will only encode characters that are not valid in HTML12 * documents, as well as non-ASCII characters.13 *14 * If a character has no equivalent entity, a numeric hexadecimal reference15 * (eg. `&#xfc;`) will be used.16 */17export function encodeHTML(input: string): string {18    return encodeHTMLTrieRe(htmlReplacer, input);19}20/**21 * Encodes all non-ASCII characters, as well as characters not valid in HTML22 * documents using HTML entities. This function will not encode characters that23 * are valid in HTML documents, such as `#`.24 *25 * If a character has no equivalent entity, a numeric hexadecimal reference26 * (eg. `&#xfc;`) will be used.27 */28export function encodeNonAsciiHTML(input: string): string {29    return encodeHTMLTrieRe(xmlReplacer, input);30}31 32function encodeHTMLTrieRe(regExp: RegExp, input: string): string {33    let returnValue = "";34    let lastIndex = 0;35    let match;36 37    while ((match = regExp.exec(input)) !== null) {38        const { index } = match;39        returnValue += input.substring(lastIndex, index);40        const char = input.charCodeAt(index);41        let next = htmlTrie.get(char);42 43        if (typeof next === "object") {44            // We are in a branch. Try to match the next char.45            if (index + 1 < input.length) {46                const nextChar = input.charCodeAt(index + 1);47                const value =48                    typeof next.n === "number"49                        ? next.n === nextChar50                            ? next.o51                            : undefined52                        : next.n.get(nextChar);53 54                if (value !== undefined) {55                    returnValue += value;56                    lastIndex = regExp.lastIndex += 1;57                    continue;58                }59            }60 61            next = next.v;62        }63 64        // We might have a tree node without a value; skip and use a numeric entity.65        if (next === undefined) {66            const cp = getCodePoint(input, index);67            returnValue += `&#x${cp.toString(16)};`;68            // Increase by 1 if we have a surrogate pair69            lastIndex = regExp.lastIndex += Number(cp !== char);70        } else {71            returnValue += next;72            lastIndex = index + 1;73        }74    }75 76    return returnValue + input.substr(lastIndex);77}78