Brunobkr/llama.cpp_AlgMor24_github

1"use strict";2Object.defineProperty(exports, "__esModule", { value: true });3exports.fromCodePoint = exports.replaceCodePoint = exports.decodeCodePoint = exports.xmlDecodeTree = exports.htmlDecodeTree = exports.EntityDecoder = exports.DecodingMode = exports.BinTrieFlags = void 0;4exports.determineBranch = determineBranch;5exports.decodeHTML = decodeHTML;6exports.decodeHTMLAttribute = decodeHTMLAttribute;7exports.decodeHTMLStrict = decodeHTMLStrict;8exports.decodeXML = decodeXML;9const decode_data_html_js_1 = require("./generated/decode-data-html.js");10const decode_data_xml_js_1 = require("./generated/decode-data-xml.js");11const decode_codepoint_js_1 = require("./decode-codepoint.js");12var CharCodes;13(function (CharCodes) {14    CharCodes[CharCodes["NUM"] = 35] = "NUM";15    CharCodes[CharCodes["SEMI"] = 59] = "SEMI";16    CharCodes[CharCodes["EQUALS"] = 61] = "EQUALS";17    CharCodes[CharCodes["ZERO"] = 48] = "ZERO";18    CharCodes[CharCodes["NINE"] = 57] = "NINE";19    CharCodes[CharCodes["LOWER_A"] = 97] = "LOWER_A";20    CharCodes[CharCodes["LOWER_F"] = 102] = "LOWER_F";21    CharCodes[CharCodes["LOWER_X"] = 120] = "LOWER_X";22    CharCodes[CharCodes["LOWER_Z"] = 122] = "LOWER_Z";23    CharCodes[CharCodes["UPPER_A"] = 65] = "UPPER_A";24    CharCodes[CharCodes["UPPER_F"] = 70] = "UPPER_F";25    CharCodes[CharCodes["UPPER_Z"] = 90] = "UPPER_Z";26})(CharCodes || (CharCodes = {}));27/** Bit that needs to be set to convert an upper case ASCII character to lower case */28const TO_LOWER_BIT = 32;29var BinTrieFlags;30(function (BinTrieFlags) {31    BinTrieFlags[BinTrieFlags["VALUE_LENGTH"] = 49152] = "VALUE_LENGTH";32    BinTrieFlags[BinTrieFlags["BRANCH_LENGTH"] = 16256] = "BRANCH_LENGTH";33    BinTrieFlags[BinTrieFlags["JUMP_TABLE"] = 127] = "JUMP_TABLE";34})(BinTrieFlags || (exports.BinTrieFlags = BinTrieFlags = {}));35function isNumber(code) {36    return code >= CharCodes.ZERO && code <= CharCodes.NINE;37}38function isHexadecimalCharacter(code) {39    return ((code >= CharCodes.UPPER_A && code <= CharCodes.UPPER_F) ||40        (code >= CharCodes.LOWER_A && code <= CharCodes.LOWER_F));41}42function isAsciiAlphaNumeric(code) {43    return ((code >= CharCodes.UPPER_A && code <= CharCodes.UPPER_Z) ||44        (code >= CharCodes.LOWER_A && code <= CharCodes.LOWER_Z) ||45        isNumber(code));46}47/**48 * Checks if the given character is a valid end character for an entity in an attribute.49 *50 * Attribute values that aren't terminated properly aren't parsed, and shouldn't lead to a parser error.51 * See the example in https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state52 */53function isEntityInAttributeInvalidEnd(code) {54    return code === CharCodes.EQUALS || isAsciiAlphaNumeric(code);55}56var EntityDecoderState;57(function (EntityDecoderState) {58    EntityDecoderState[EntityDecoderState["EntityStart"] = 0] = "EntityStart";59    EntityDecoderState[EntityDecoderState["NumericStart"] = 1] = "NumericStart";60    EntityDecoderState[EntityDecoderState["NumericDecimal"] = 2] = "NumericDecimal";61    EntityDecoderState[EntityDecoderState["NumericHex"] = 3] = "NumericHex";62    EntityDecoderState[EntityDecoderState["NamedEntity"] = 4] = "NamedEntity";63})(EntityDecoderState || (EntityDecoderState = {}));64var DecodingMode;65(function (DecodingMode) {66    /** Entities in text nodes that can end with any character. */67    DecodingMode[DecodingMode["Legacy"] = 0] = "Legacy";68    /** Only allow entities terminated with a semicolon. */69    DecodingMode[DecodingMode["Strict"] = 1] = "Strict";70    /** Entities in attributes have limitations on ending characters. */71    DecodingMode[DecodingMode["Attribute"] = 2] = "Attribute";72})(DecodingMode || (exports.DecodingMode = DecodingMode = {}));73/**74 * Token decoder with support of writing partial entities.75 */76class EntityDecoder {77    constructor(78    /** The tree used to decode entities. */79    decodeTree, 80    /**81     * The function that is called when a codepoint is decoded.82     *83     * For multi-byte named entities, this will be called multiple times,84     * with the second codepoint, and the same `consumed` value.85     *86     * @param codepoint The decoded codepoint.87     * @param consumed The number of bytes consumed by the decoder.88     */89    emitCodePoint, 90    /** An object that is used to produce errors. */91    errors) {92        this.decodeTree = decodeTree;93        this.emitCodePoint = emitCodePoint;94        this.errors = errors;95        /** The current state of the decoder. */96        this.state = EntityDecoderState.EntityStart;97        /** Characters that were consumed while parsing an entity. */98        this.consumed = 1;99        /**100         * The result of the entity.101         *102         * Either the result index of a numeric entity, or the codepoint of a103         * numeric entity.104         */105        this.result = 0;106        /** The current index in the decode tree. */107        this.treeIndex = 0;108        /** The number of characters that were consumed in excess. */109        this.excess = 1;110        /** The mode in which the decoder is operating. */111        this.decodeMode = DecodingMode.Strict;112    }113    /** Resets the instance to make it reusable. */114    startEntity(decodeMode) {115        this.decodeMode = decodeMode;116        this.state = EntityDecoderState.EntityStart;117        this.result = 0;118        this.treeIndex = 0;119        this.excess = 1;120        this.consumed = 1;121    }122    /**123     * Write an entity to the decoder. This can be called multiple times with partial entities.124     * If the entity is incomplete, the decoder will return -1.125     *126     * Mirrors the implementation of `getDecoder`, but with the ability to stop decoding if the127     * entity is incomplete, and resume when the next string is written.128     *129     * @param input The string containing the entity (or a continuation of the entity).130     * @param offset The offset at which the entity begins. Should be 0 if this is not the first call.131     * @returns The number of characters that were consumed, or -1 if the entity is incomplete.132     */133    write(input, offset) {134        switch (this.state) {135            case EntityDecoderState.EntityStart: {136                if (input.charCodeAt(offset) === CharCodes.NUM) {137                    this.state = EntityDecoderState.NumericStart;138                    this.consumed += 1;139                    return this.stateNumericStart(input, offset + 1);140                }141                this.state = EntityDecoderState.NamedEntity;142                return this.stateNamedEntity(input, offset);143            }144            case EntityDecoderState.NumericStart: {145                return this.stateNumericStart(input, offset);146            }147            case EntityDecoderState.NumericDecimal: {148                return this.stateNumericDecimal(input, offset);149            }150            case EntityDecoderState.NumericHex: {151                return this.stateNumericHex(input, offset);152            }153            case EntityDecoderState.NamedEntity: {154                return this.stateNamedEntity(input, offset);155            }156        }157    }158    /**159     * Switches between the numeric decimal and hexadecimal states.160     *161     * Equivalent to the `Numeric character reference state` in the HTML spec.162     *163     * @param input The string containing the entity (or a continuation of the entity).164     * @param offset The current offset.165     * @returns The number of characters that were consumed, or -1 if the entity is incomplete.166     */167    stateNumericStart(input, offset) {168        if (offset >= input.length) {169            return -1;170        }171        if ((input.charCodeAt(offset) | TO_LOWER_BIT) === CharCodes.LOWER_X) {172            this.state = EntityDecoderState.NumericHex;173            this.consumed += 1;174            return this.stateNumericHex(input, offset + 1);175        }176        this.state = EntityDecoderState.NumericDecimal;177        return this.stateNumericDecimal(input, offset);178    }179    addToNumericResult(input, start, end, base) {180        if (start !== end) {181            const digitCount = end - start;182            this.result =183                this.result * Math.pow(base, digitCount) +184                    Number.parseInt(input.substr(start, digitCount), base);185            this.consumed += digitCount;186        }187    }188    /**189     * Parses a hexadecimal numeric entity.190     *191     * Equivalent to the `Hexademical character reference state` in the HTML spec.192     *193     * @param input The string containing the entity (or a continuation of the entity).194     * @param offset The current offset.195     * @returns The number of characters that were consumed, or -1 if the entity is incomplete.196     */197    stateNumericHex(input, offset) {198        const startIndex = offset;199        while (offset < input.length) {200            const char = input.charCodeAt(offset);201            if (isNumber(char) || isHexadecimalCharacter(char)) {202                offset += 1;203            }204            else {205                this.addToNumericResult(input, startIndex, offset, 16);206                return this.emitNumericEntity(char, 3);207            }208        }209        this.addToNumericResult(input, startIndex, offset, 16);210        return -1;211    }212    /**213     * Parses a decimal numeric entity.214     *215     * Equivalent to the `Decimal character reference state` in the HTML spec.216     *217     * @param input The string containing the entity (or a continuation of the entity).218     * @param offset The current offset.219     * @returns The number of characters that were consumed, or -1 if the entity is incomplete.220     */221    stateNumericDecimal(input, offset) {222        const startIndex = offset;223        while (offset < input.length) {224            const char = input.charCodeAt(offset);225            if (isNumber(char)) {226                offset += 1;227            }228            else {229                this.addToNumericResult(input, startIndex, offset, 10);230                return this.emitNumericEntity(char, 2);231            }232        }233        this.addToNumericResult(input, startIndex, offset, 10);234        return -1;235    }236    /**237     * Validate and emit a numeric entity.238     *239     * Implements the logic from the `Hexademical character reference start240     * state` and `Numeric character reference end state` in the HTML spec.241     *242     * @param lastCp The last code point of the entity. Used to see if the243     *               entity was terminated with a semicolon.244     * @param expectedLength The minimum number of characters that should be245     *                       consumed. Used to validate that at least one digit246     *                       was consumed.247     * @returns The number of characters that were consumed.248     */249    emitNumericEntity(lastCp, expectedLength) {250        var _a;251        // Ensure we consumed at least one digit.252        if (this.consumed <= expectedLength) {253            (_a = this.errors) === null || _a === void 0 ? void 0 : _a.absenceOfDigitsInNumericCharacterReference(this.consumed);254            return 0;255        }256        // Figure out if this is a legit end of the entity257        if (lastCp === CharCodes.SEMI) {258            this.consumed += 1;259        }260        else if (this.decodeMode === DecodingMode.Strict) {261            return 0;262        }263        this.emitCodePoint((0, decode_codepoint_js_1.replaceCodePoint)(this.result), this.consumed);264        if (this.errors) {265            if (lastCp !== CharCodes.SEMI) {266                this.errors.missingSemicolonAfterCharacterReference();267            }268            this.errors.validateNumericCharacterReference(this.result);269        }270        return this.consumed;271    }272    /**273     * Parses a named entity.274     *275     * Equivalent to the `Named character reference state` in the HTML spec.276     *277     * @param input The string containing the entity (or a continuation of the entity).278     * @param offset The current offset.279     * @returns The number of characters that were consumed, or -1 if the entity is incomplete.280     */281    stateNamedEntity(input, offset) {282        const { decodeTree } = this;283        let current = decodeTree[this.treeIndex];284        // The mask is the number of bytes of the value, including the current byte.285        let valueLength = (current & BinTrieFlags.VALUE_LENGTH) >> 14;286        for (; offset < input.length; offset++, this.excess++) {287            const char = input.charCodeAt(offset);288            this.treeIndex = determineBranch(decodeTree, current, this.treeIndex + Math.max(1, valueLength), char);289            if (this.treeIndex < 0) {290                return this.result === 0 ||291                    // If we are parsing an attribute292                    (this.decodeMode === DecodingMode.Attribute &&293                        // We shouldn't have consumed any characters after the entity,294                        (valueLength === 0 ||295                            // And there should be no invalid characters.296                            isEntityInAttributeInvalidEnd(char)))297                    ? 0298                    : this.emitNotTerminatedNamedEntity();299            }300            current = decodeTree[this.treeIndex];301            valueLength = (current & BinTrieFlags.VALUE_LENGTH) >> 14;302            // If the branch is a value, store it and continue303            if (valueLength !== 0) {304                // If the entity is terminated by a semicolon, we are done.305                if (char === CharCodes.SEMI) {306                    return this.emitNamedEntityData(this.treeIndex, valueLength, this.consumed + this.excess);307                }308                // If we encounter a non-terminated (legacy) entity while parsing strictly, then ignore it.309                if (this.decodeMode !== DecodingMode.Strict) {310                    this.result = this.treeIndex;311                    this.consumed += this.excess;312                    this.excess = 0;313                }314            }315        }316        return -1;317    }318    /**319     * Emit a named entity that was not terminated with a semicolon.320     *321     * @returns The number of characters consumed.322     */323    emitNotTerminatedNamedEntity() {324        var _a;325        const { result, decodeTree } = this;326        const valueLength = (decodeTree[result] & BinTrieFlags.VALUE_LENGTH) >> 14;327        this.emitNamedEntityData(result, valueLength, this.consumed);328        (_a = this.errors) === null || _a === void 0 ? void 0 : _a.missingSemicolonAfterCharacterReference();329        return this.consumed;330    }331    /**332     * Emit a named entity.333     *334     * @param result The index of the entity in the decode tree.335     * @param valueLength The number of bytes in the entity.336     * @param consumed The number of characters consumed.337     *338     * @returns The number of characters consumed.339     */340    emitNamedEntityData(result, valueLength, consumed) {341        const { decodeTree } = this;342        this.emitCodePoint(valueLength === 1343            ? decodeTree[result] & ~BinTrieFlags.VALUE_LENGTH344            : decodeTree[result + 1], consumed);345        if (valueLength === 3) {346            // For multi-byte values, we need to emit the second byte.347            this.emitCodePoint(decodeTree[result + 2], consumed);348        }349        return consumed;350    }351    /**352     * Signal to the parser that the end of the input was reached.353     *354     * Remaining data will be emitted and relevant errors will be produced.355     *356     * @returns The number of characters consumed.357     */358    end() {359        var _a;360        switch (this.state) {361            case EntityDecoderState.NamedEntity: {362                // Emit a named entity if we have one.363                return this.result !== 0 &&364                    (this.decodeMode !== DecodingMode.Attribute ||365                        this.result === this.treeIndex)366                    ? this.emitNotTerminatedNamedEntity()367                    : 0;368            }369            // Otherwise, emit a numeric entity if we have one.370            case EntityDecoderState.NumericDecimal: {371                return this.emitNumericEntity(0, 2);372            }373            case EntityDecoderState.NumericHex: {374                return this.emitNumericEntity(0, 3);375            }376            case EntityDecoderState.NumericStart: {377                (_a = this.errors) === null || _a === void 0 ? void 0 : _a.absenceOfDigitsInNumericCharacterReference(this.consumed);378                return 0;379            }380            case EntityDecoderState.EntityStart: {381                // Return 0 if we have no entity.382                return 0;383            }384        }385    }386}387exports.EntityDecoder = EntityDecoder;388/**389 * Creates a function that decodes entities in a string.390 *391 * @param decodeTree The decode tree.392 * @returns A function that decodes entities in a string.393 */394function getDecoder(decodeTree) {395    let returnValue = "";396    const decoder = new EntityDecoder(decodeTree, (data) => (returnValue += (0, decode_codepoint_js_1.fromCodePoint)(data)));397    return function decodeWithTrie(input, decodeMode) {398        let lastIndex = 0;399        let offset = 0;400        while ((offset = input.indexOf("&", offset)) >= 0) {401            returnValue += input.slice(lastIndex, offset);402            decoder.startEntity(decodeMode);403            const length = decoder.write(input, 404            // Skip the "&"405            offset + 1);406            if (length < 0) {407                lastIndex = offset + decoder.end();408                break;409            }410            lastIndex = offset + length;411            // If `length` is 0, skip the current `&` and continue.412            offset = length === 0 ? lastIndex + 1 : lastIndex;413        }414        const result = returnValue + input.slice(lastIndex);415        // Make sure we don't keep a reference to the final string.416        returnValue = "";417        return result;418    };419}420/**421 * Determines the branch of the current node that is taken given the current422 * character. This function is used to traverse the trie.423 *424 * @param decodeTree The trie.425 * @param current The current node.426 * @param nodeIdx The index right after the current node and its value.427 * @param char The current character.428 * @returns The index of the next node, or -1 if no branch is taken.429 */430function determineBranch(decodeTree, current, nodeIndex, char) {431    const branchCount = (current & BinTrieFlags.BRANCH_LENGTH) >> 7;432    const jumpOffset = current & BinTrieFlags.JUMP_TABLE;433    // Case 1: Single branch encoded in jump offset434    if (branchCount === 0) {435        return jumpOffset !== 0 && char === jumpOffset ? nodeIndex : -1;436    }437    // Case 2: Multiple branches encoded in jump table438    if (jumpOffset) {439        const value = char - jumpOffset;440        return value < 0 || value >= branchCount441            ? -1442            : decodeTree[nodeIndex + value] - 1;443    }444    // Case 3: Multiple branches encoded in dictionary445    // Binary search for the character.446    let lo = nodeIndex;447    let hi = lo + branchCount - 1;448    while (lo <= hi) {449        const mid = (lo + hi) >>> 1;450        const midValue = decodeTree[mid];451        if (midValue < char) {452            lo = mid + 1;453        }454        else if (midValue > char) {455            hi = mid - 1;456        }457        else {458            return decodeTree[mid + branchCount];459        }460    }461    return -1;462}463const htmlDecoder = /* #__PURE__ */ getDecoder(decode_data_html_js_1.htmlDecodeTree);464const xmlDecoder = /* #__PURE__ */ getDecoder(decode_data_xml_js_1.xmlDecodeTree);465/**466 * Decodes an HTML string.467 *468 * @param htmlString The string to decode.469 * @param mode The decoding mode.470 * @returns The decoded string.471 */472function decodeHTML(htmlString, mode = DecodingMode.Legacy) {473    return htmlDecoder(htmlString, mode);474}475/**476 * Decodes an HTML string in an attribute.477 *478 * @param htmlAttribute The string to decode.479 * @returns The decoded string.480 */481function decodeHTMLAttribute(htmlAttribute) {482    return htmlDecoder(htmlAttribute, DecodingMode.Attribute);483}484/**485 * Decodes an HTML string, requiring all entities to be terminated by a semicolon.486 *487 * @param htmlString The string to decode.488 * @returns The decoded string.489 */490function decodeHTMLStrict(htmlString) {491    return htmlDecoder(htmlString, DecodingMode.Strict);492}493/**494 * Decodes an XML string, requiring all entities to be terminated by a semicolon.495 *496 * @param xmlString The string to decode.497 * @returns The decoded string.498 */499function decodeXML(xmlString) {500    return xmlDecoder(xmlString, DecodingMode.Strict);501}502// Re-export for use by eg. htmlparser2503var decode_data_html_js_2 = require("./generated/decode-data-html.js");504Object.defineProperty(exports, "htmlDecodeTree", { enumerable: true, get: function () { return decode_data_html_js_2.htmlDecodeTree; } });505var decode_data_xml_js_2 = require("./generated/decode-data-xml.js");506Object.defineProperty(exports, "xmlDecodeTree", { enumerable: true, get: function () { return decode_data_xml_js_2.xmlDecodeTree; } });507var decode_codepoint_js_2 = require("./decode-codepoint.js");508Object.defineProperty(exports, "decodeCodePoint", { enumerable: true, get: function () { return decode_codepoint_js_2.decodeCodePoint; } });509Object.defineProperty(exports, "replaceCodePoint", { enumerable: true, get: function () { return decode_codepoint_js_2.replaceCodePoint; } });510Object.defineProperty(exports, "fromCodePoint", { enumerable: true, get: function () { return decode_codepoint_js_2.fromCodePoint; } });511//# sourceMappingURL=decode.js.map