Brunobkr/llama.cpp_AlgMor24_github

1"use strict";2Object.defineProperty(exports, "__esModule", { value: true });3exports.Tokenizer = exports.TokenizerMode = void 0;4const preprocessor_js_1 = require("./preprocessor.js");5const unicode_js_1 = require("../common/unicode.js");6const token_js_1 = require("../common/token.js");7const decode_1 = require("entities/decode");8const error_codes_js_1 = require("../common/error-codes.js");9const html_js_1 = require("../common/html.js");10//States11var State;12(function (State) {13    State[State["DATA"] = 0] = "DATA";14    State[State["RCDATA"] = 1] = "RCDATA";15    State[State["RAWTEXT"] = 2] = "RAWTEXT";16    State[State["SCRIPT_DATA"] = 3] = "SCRIPT_DATA";17    State[State["PLAINTEXT"] = 4] = "PLAINTEXT";18    State[State["TAG_OPEN"] = 5] = "TAG_OPEN";19    State[State["END_TAG_OPEN"] = 6] = "END_TAG_OPEN";20    State[State["TAG_NAME"] = 7] = "TAG_NAME";21    State[State["RCDATA_LESS_THAN_SIGN"] = 8] = "RCDATA_LESS_THAN_SIGN";22    State[State["RCDATA_END_TAG_OPEN"] = 9] = "RCDATA_END_TAG_OPEN";23    State[State["RCDATA_END_TAG_NAME"] = 10] = "RCDATA_END_TAG_NAME";24    State[State["RAWTEXT_LESS_THAN_SIGN"] = 11] = "RAWTEXT_LESS_THAN_SIGN";25    State[State["RAWTEXT_END_TAG_OPEN"] = 12] = "RAWTEXT_END_TAG_OPEN";26    State[State["RAWTEXT_END_TAG_NAME"] = 13] = "RAWTEXT_END_TAG_NAME";27    State[State["SCRIPT_DATA_LESS_THAN_SIGN"] = 14] = "SCRIPT_DATA_LESS_THAN_SIGN";28    State[State["SCRIPT_DATA_END_TAG_OPEN"] = 15] = "SCRIPT_DATA_END_TAG_OPEN";29    State[State["SCRIPT_DATA_END_TAG_NAME"] = 16] = "SCRIPT_DATA_END_TAG_NAME";30    State[State["SCRIPT_DATA_ESCAPE_START"] = 17] = "SCRIPT_DATA_ESCAPE_START";31    State[State["SCRIPT_DATA_ESCAPE_START_DASH"] = 18] = "SCRIPT_DATA_ESCAPE_START_DASH";32    State[State["SCRIPT_DATA_ESCAPED"] = 19] = "SCRIPT_DATA_ESCAPED";33    State[State["SCRIPT_DATA_ESCAPED_DASH"] = 20] = "SCRIPT_DATA_ESCAPED_DASH";34    State[State["SCRIPT_DATA_ESCAPED_DASH_DASH"] = 21] = "SCRIPT_DATA_ESCAPED_DASH_DASH";35    State[State["SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN"] = 22] = "SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN";36    State[State["SCRIPT_DATA_ESCAPED_END_TAG_OPEN"] = 23] = "SCRIPT_DATA_ESCAPED_END_TAG_OPEN";37    State[State["SCRIPT_DATA_ESCAPED_END_TAG_NAME"] = 24] = "SCRIPT_DATA_ESCAPED_END_TAG_NAME";38    State[State["SCRIPT_DATA_DOUBLE_ESCAPE_START"] = 25] = "SCRIPT_DATA_DOUBLE_ESCAPE_START";39    State[State["SCRIPT_DATA_DOUBLE_ESCAPED"] = 26] = "SCRIPT_DATA_DOUBLE_ESCAPED";40    State[State["SCRIPT_DATA_DOUBLE_ESCAPED_DASH"] = 27] = "SCRIPT_DATA_DOUBLE_ESCAPED_DASH";41    State[State["SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH"] = 28] = "SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH";42    State[State["SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN"] = 29] = "SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN";43    State[State["SCRIPT_DATA_DOUBLE_ESCAPE_END"] = 30] = "SCRIPT_DATA_DOUBLE_ESCAPE_END";44    State[State["BEFORE_ATTRIBUTE_NAME"] = 31] = "BEFORE_ATTRIBUTE_NAME";45    State[State["ATTRIBUTE_NAME"] = 32] = "ATTRIBUTE_NAME";46    State[State["AFTER_ATTRIBUTE_NAME"] = 33] = "AFTER_ATTRIBUTE_NAME";47    State[State["BEFORE_ATTRIBUTE_VALUE"] = 34] = "BEFORE_ATTRIBUTE_VALUE";48    State[State["ATTRIBUTE_VALUE_DOUBLE_QUOTED"] = 35] = "ATTRIBUTE_VALUE_DOUBLE_QUOTED";49    State[State["ATTRIBUTE_VALUE_SINGLE_QUOTED"] = 36] = "ATTRIBUTE_VALUE_SINGLE_QUOTED";50    State[State["ATTRIBUTE_VALUE_UNQUOTED"] = 37] = "ATTRIBUTE_VALUE_UNQUOTED";51    State[State["AFTER_ATTRIBUTE_VALUE_QUOTED"] = 38] = "AFTER_ATTRIBUTE_VALUE_QUOTED";52    State[State["SELF_CLOSING_START_TAG"] = 39] = "SELF_CLOSING_START_TAG";53    State[State["BOGUS_COMMENT"] = 40] = "BOGUS_COMMENT";54    State[State["MARKUP_DECLARATION_OPEN"] = 41] = "MARKUP_DECLARATION_OPEN";55    State[State["COMMENT_START"] = 42] = "COMMENT_START";56    State[State["COMMENT_START_DASH"] = 43] = "COMMENT_START_DASH";57    State[State["COMMENT"] = 44] = "COMMENT";58    State[State["COMMENT_LESS_THAN_SIGN"] = 45] = "COMMENT_LESS_THAN_SIGN";59    State[State["COMMENT_LESS_THAN_SIGN_BANG"] = 46] = "COMMENT_LESS_THAN_SIGN_BANG";60    State[State["COMMENT_LESS_THAN_SIGN_BANG_DASH"] = 47] = "COMMENT_LESS_THAN_SIGN_BANG_DASH";61    State[State["COMMENT_LESS_THAN_SIGN_BANG_DASH_DASH"] = 48] = "COMMENT_LESS_THAN_SIGN_BANG_DASH_DASH";62    State[State["COMMENT_END_DASH"] = 49] = "COMMENT_END_DASH";63    State[State["COMMENT_END"] = 50] = "COMMENT_END";64    State[State["COMMENT_END_BANG"] = 51] = "COMMENT_END_BANG";65    State[State["DOCTYPE"] = 52] = "DOCTYPE";66    State[State["BEFORE_DOCTYPE_NAME"] = 53] = "BEFORE_DOCTYPE_NAME";67    State[State["DOCTYPE_NAME"] = 54] = "DOCTYPE_NAME";68    State[State["AFTER_DOCTYPE_NAME"] = 55] = "AFTER_DOCTYPE_NAME";69    State[State["AFTER_DOCTYPE_PUBLIC_KEYWORD"] = 56] = "AFTER_DOCTYPE_PUBLIC_KEYWORD";70    State[State["BEFORE_DOCTYPE_PUBLIC_IDENTIFIER"] = 57] = "BEFORE_DOCTYPE_PUBLIC_IDENTIFIER";71    State[State["DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED"] = 58] = "DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED";72    State[State["DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED"] = 59] = "DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED";73    State[State["AFTER_DOCTYPE_PUBLIC_IDENTIFIER"] = 60] = "AFTER_DOCTYPE_PUBLIC_IDENTIFIER";74    State[State["BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS"] = 61] = "BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS";75    State[State["AFTER_DOCTYPE_SYSTEM_KEYWORD"] = 62] = "AFTER_DOCTYPE_SYSTEM_KEYWORD";76    State[State["BEFORE_DOCTYPE_SYSTEM_IDENTIFIER"] = 63] = "BEFORE_DOCTYPE_SYSTEM_IDENTIFIER";77    State[State["DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED"] = 64] = "DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED";78    State[State["DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED"] = 65] = "DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED";79    State[State["AFTER_DOCTYPE_SYSTEM_IDENTIFIER"] = 66] = "AFTER_DOCTYPE_SYSTEM_IDENTIFIER";80    State[State["BOGUS_DOCTYPE"] = 67] = "BOGUS_DOCTYPE";81    State[State["CDATA_SECTION"] = 68] = "CDATA_SECTION";82    State[State["CDATA_SECTION_BRACKET"] = 69] = "CDATA_SECTION_BRACKET";83    State[State["CDATA_SECTION_END"] = 70] = "CDATA_SECTION_END";84    State[State["CHARACTER_REFERENCE"] = 71] = "CHARACTER_REFERENCE";85    State[State["AMBIGUOUS_AMPERSAND"] = 72] = "AMBIGUOUS_AMPERSAND";86})(State || (State = {}));87//Tokenizer initial states for different modes88exports.TokenizerMode = {89    DATA: State.DATA,90    RCDATA: State.RCDATA,91    RAWTEXT: State.RAWTEXT,92    SCRIPT_DATA: State.SCRIPT_DATA,93    PLAINTEXT: State.PLAINTEXT,94    CDATA_SECTION: State.CDATA_SECTION,95};96//Utils97//OPTIMIZATION: these utility functions should not be moved out of this module. V8 Crankshaft will not inline98//this functions if they will be situated in another module due to context switch.99//Always perform inlining check before modifying this functions ('node --trace-inlining').100function isAsciiDigit(cp) {101    return cp >= unicode_js_1.CODE_POINTS.DIGIT_0 && cp <= unicode_js_1.CODE_POINTS.DIGIT_9;102}103function isAsciiUpper(cp) {104    return cp >= unicode_js_1.CODE_POINTS.LATIN_CAPITAL_A && cp <= unicode_js_1.CODE_POINTS.LATIN_CAPITAL_Z;105}106function isAsciiLower(cp) {107    return cp >= unicode_js_1.CODE_POINTS.LATIN_SMALL_A && cp <= unicode_js_1.CODE_POINTS.LATIN_SMALL_Z;108}109function isAsciiLetter(cp) {110    return isAsciiLower(cp) || isAsciiUpper(cp);111}112function isAsciiAlphaNumeric(cp) {113    return isAsciiLetter(cp) || isAsciiDigit(cp);114}115function toAsciiLower(cp) {116    return cp + 32;117}118function isWhitespace(cp) {119    return cp === unicode_js_1.CODE_POINTS.SPACE || cp === unicode_js_1.CODE_POINTS.LINE_FEED || cp === unicode_js_1.CODE_POINTS.TABULATION || cp === unicode_js_1.CODE_POINTS.FORM_FEED;120}121function isScriptDataDoubleEscapeSequenceEnd(cp) {122    return isWhitespace(cp) || cp === unicode_js_1.CODE_POINTS.SOLIDUS || cp === unicode_js_1.CODE_POINTS.GREATER_THAN_SIGN;123}124function getErrorForNumericCharacterReference(code) {125    if (code === unicode_js_1.CODE_POINTS.NULL) {126        return error_codes_js_1.ERR.nullCharacterReference;127    }128    else if (code > 1114111) {129        return error_codes_js_1.ERR.characterReferenceOutsideUnicodeRange;130    }131    else if ((0, unicode_js_1.isSurrogate)(code)) {132        return error_codes_js_1.ERR.surrogateCharacterReference;133    }134    else if ((0, unicode_js_1.isUndefinedCodePoint)(code)) {135        return error_codes_js_1.ERR.noncharacterCharacterReference;136    }137    else if ((0, unicode_js_1.isControlCodePoint)(code) || code === unicode_js_1.CODE_POINTS.CARRIAGE_RETURN) {138        return error_codes_js_1.ERR.controlCharacterReference;139    }140    return null;141}142//Tokenizer143class Tokenizer {144    constructor(options, handler) {145        this.options = options;146        this.handler = handler;147        this.paused = false;148        /** Ensures that the parsing loop isn't run multiple times at once. */149        this.inLoop = false;150        /**151         * Indicates that the current adjusted node exists, is not an element in the HTML namespace,152         * and that it is not an integration point for either MathML or HTML.153         *154         * @see {@link https://html.spec.whatwg.org/multipage/parsing.html#tree-construction}155         */156        this.inForeignNode = false;157        this.lastStartTagName = '';158        this.active = false;159        this.state = State.DATA;160        this.returnState = State.DATA;161        this.entityStartPos = 0;162        this.consumedAfterSnapshot = -1;163        this.currentCharacterToken = null;164        this.currentToken = null;165        this.currentAttr = { name: '', value: '' };166        this.preprocessor = new preprocessor_js_1.Preprocessor(handler);167        this.currentLocation = this.getCurrentLocation(-1);168        this.entityDecoder = new decode_1.EntityDecoder(decode_1.htmlDecodeTree, (cp, consumed) => {169            // Note: Set `pos` _before_ flushing, as flushing might drop170            // the current chunk and invalidate `entityStartPos`.171            this.preprocessor.pos = this.entityStartPos + consumed - 1;172            this._flushCodePointConsumedAsCharacterReference(cp);173        }, handler.onParseError174            ? {175                missingSemicolonAfterCharacterReference: () => {176                    this._err(error_codes_js_1.ERR.missingSemicolonAfterCharacterReference, 1);177                },178                absenceOfDigitsInNumericCharacterReference: (consumed) => {179                    this._err(error_codes_js_1.ERR.absenceOfDigitsInNumericCharacterReference, this.entityStartPos - this.preprocessor.pos + consumed);180                },181                validateNumericCharacterReference: (code) => {182                    const error = getErrorForNumericCharacterReference(code);183                    if (error)184                        this._err(error, 1);185                },186            }187            : undefined);188    }189    //Errors190    _err(code, cpOffset = 0) {191        var _a, _b;192        (_b = (_a = this.handler).onParseError) === null || _b === void 0 ? void 0 : _b.call(_a, this.preprocessor.getError(code, cpOffset));193    }194    // NOTE: `offset` may never run across line boundaries.195    getCurrentLocation(offset) {196        if (!this.options.sourceCodeLocationInfo) {197            return null;198        }199        return {200            startLine: this.preprocessor.line,201            startCol: this.preprocessor.col - offset,202            startOffset: this.preprocessor.offset - offset,203            endLine: -1,204            endCol: -1,205            endOffset: -1,206        };207    }208    _runParsingLoop() {209        if (this.inLoop)210            return;211        this.inLoop = true;212        while (this.active && !this.paused) {213            this.consumedAfterSnapshot = 0;214            const cp = this._consume();215            if (!this._ensureHibernation()) {216                this._callState(cp);217            }218        }219        this.inLoop = false;220    }221    //API222    pause() {223        this.paused = true;224    }225    resume(writeCallback) {226        if (!this.paused) {227            throw new Error('Parser was already resumed');228        }229        this.paused = false;230        // Necessary for synchronous resume.231        if (this.inLoop)232            return;233        this._runParsingLoop();234        if (!this.paused) {235            writeCallback === null || writeCallback === void 0 ? void 0 : writeCallback();236        }237    }238    write(chunk, isLastChunk, writeCallback) {239        this.active = true;240        this.preprocessor.write(chunk, isLastChunk);241        this._runParsingLoop();242        if (!this.paused) {243            writeCallback === null || writeCallback === void 0 ? void 0 : writeCallback();244        }245    }246    insertHtmlAtCurrentPos(chunk) {247        this.active = true;248        this.preprocessor.insertHtmlAtCurrentPos(chunk);249        this._runParsingLoop();250    }251    //Hibernation252    _ensureHibernation() {253        if (this.preprocessor.endOfChunkHit) {254            this.preprocessor.retreat(this.consumedAfterSnapshot);255            this.consumedAfterSnapshot = 0;256            this.active = false;257            return true;258        }259        return false;260    }261    //Consumption262    _consume() {263        this.consumedAfterSnapshot++;264        return this.preprocessor.advance();265    }266    _advanceBy(count) {267        this.consumedAfterSnapshot += count;268        for (let i = 0; i < count; i++) {269            this.preprocessor.advance();270        }271    }272    _consumeSequenceIfMatch(pattern, caseSensitive) {273        if (this.preprocessor.startsWith(pattern, caseSensitive)) {274            // We will already have consumed one character before calling this method.275            this._advanceBy(pattern.length - 1);276            return true;277        }278        return false;279    }280    //Token creation281    _createStartTagToken() {282        this.currentToken = {283            type: token_js_1.TokenType.START_TAG,284            tagName: '',285            tagID: html_js_1.TAG_ID.UNKNOWN,286            selfClosing: false,287            ackSelfClosing: false,288            attrs: [],289            location: this.getCurrentLocation(1),290        };291    }292    _createEndTagToken() {293        this.currentToken = {294            type: token_js_1.TokenType.END_TAG,295            tagName: '',296            tagID: html_js_1.TAG_ID.UNKNOWN,297            selfClosing: false,298            ackSelfClosing: false,299            attrs: [],300            location: this.getCurrentLocation(2),301        };302    }303    _createCommentToken(offset) {304        this.currentToken = {305            type: token_js_1.TokenType.COMMENT,306            data: '',307            location: this.getCurrentLocation(offset),308        };309    }310    _createDoctypeToken(initialName) {311        this.currentToken = {312            type: token_js_1.TokenType.DOCTYPE,313            name: initialName,314            forceQuirks: false,315            publicId: null,316            systemId: null,317            location: this.currentLocation,318        };319    }320    _createCharacterToken(type, chars) {321        this.currentCharacterToken = {322            type,323            chars,324            location: this.currentLocation,325        };326    }327    //Tag attributes328    _createAttr(attrNameFirstCh) {329        this.currentAttr = {330            name: attrNameFirstCh,331            value: '',332        };333        this.currentLocation = this.getCurrentLocation(0);334    }335    _leaveAttrName() {336        var _a;337        var _b;338        const token = this.currentToken;339        if ((0, token_js_1.getTokenAttr)(token, this.currentAttr.name) === null) {340            token.attrs.push(this.currentAttr);341            if (token.location && this.currentLocation) {342                const attrLocations = ((_a = (_b = token.location).attrs) !== null && _a !== void 0 ? _a : (_b.attrs = Object.create(null)));343                attrLocations[this.currentAttr.name] = this.currentLocation;344                // Set end location345                this._leaveAttrValue();346            }347        }348        else {349            this._err(error_codes_js_1.ERR.duplicateAttribute);350        }351    }352    _leaveAttrValue() {353        if (this.currentLocation) {354            this.currentLocation.endLine = this.preprocessor.line;355            this.currentLocation.endCol = this.preprocessor.col;356            this.currentLocation.endOffset = this.preprocessor.offset;357        }358    }359    //Token emission360    prepareToken(ct) {361        this._emitCurrentCharacterToken(ct.location);362        this.currentToken = null;363        if (ct.location) {364            ct.location.endLine = this.preprocessor.line;365            ct.location.endCol = this.preprocessor.col + 1;366            ct.location.endOffset = this.preprocessor.offset + 1;367        }368        this.currentLocation = this.getCurrentLocation(-1);369    }370    emitCurrentTagToken() {371        const ct = this.currentToken;372        this.prepareToken(ct);373        ct.tagID = (0, html_js_1.getTagID)(ct.tagName);374        if (ct.type === token_js_1.TokenType.START_TAG) {375            this.lastStartTagName = ct.tagName;376            this.handler.onStartTag(ct);377        }378        else {379            if (ct.attrs.length > 0) {380                this._err(error_codes_js_1.ERR.endTagWithAttributes);381            }382            if (ct.selfClosing) {383                this._err(error_codes_js_1.ERR.endTagWithTrailingSolidus);384            }385            this.handler.onEndTag(ct);386        }387        this.preprocessor.dropParsedChunk();388    }389    emitCurrentComment(ct) {390        this.prepareToken(ct);391        this.handler.onComment(ct);392        this.preprocessor.dropParsedChunk();393    }394    emitCurrentDoctype(ct) {395        this.prepareToken(ct);396        this.handler.onDoctype(ct);397        this.preprocessor.dropParsedChunk();398    }399    _emitCurrentCharacterToken(nextLocation) {400        if (this.currentCharacterToken) {401            //NOTE: if we have a pending character token, make it's end location equal to the402            //current token's start location.403            if (nextLocation && this.currentCharacterToken.location) {404                this.currentCharacterToken.location.endLine = nextLocation.startLine;405                this.currentCharacterToken.location.endCol = nextLocation.startCol;406                this.currentCharacterToken.location.endOffset = nextLocation.startOffset;407            }408            switch (this.currentCharacterToken.type) {409                case token_js_1.TokenType.CHARACTER: {410                    this.handler.onCharacter(this.currentCharacterToken);411                    break;412                }413                case token_js_1.TokenType.NULL_CHARACTER: {414                    this.handler.onNullCharacter(this.currentCharacterToken);415                    break;416                }417                case token_js_1.TokenType.WHITESPACE_CHARACTER: {418                    this.handler.onWhitespaceCharacter(this.currentCharacterToken);419                    break;420                }421            }422            this.currentCharacterToken = null;423        }424    }425    _emitEOFToken() {426        const location = this.getCurrentLocation(0);427        if (location) {428            location.endLine = location.startLine;429            location.endCol = location.startCol;430            location.endOffset = location.startOffset;431        }432        this._emitCurrentCharacterToken(location);433        this.handler.onEof({ type: token_js_1.TokenType.EOF, location });434        this.active = false;435    }436    //Characters emission437    //OPTIMIZATION: The specification uses only one type of character token (one token per character).438    //This causes a huge memory overhead and a lot of unnecessary parser loops. parse5 uses 3 groups of characters.439    //If we have a sequence of characters that belong to the same group, the parser can process it440    //as a single solid character token.441    //So, there are 3 types of character tokens in parse5:442    //1)TokenType.NULL_CHARACTER - \u0000-character sequences (e.g. '\u0000\u0000\u0000')443    //2)TokenType.WHITESPACE_CHARACTER - any whitespace/new-line character sequences (e.g. '\n  \r\t   \f')444    //3)TokenType.CHARACTER - any character sequence which don't belong to groups 1 and 2 (e.g. 'abcdef1234@@#$%^')445    _appendCharToCurrentCharacterToken(type, ch) {446        if (this.currentCharacterToken) {447            if (this.currentCharacterToken.type === type) {448                this.currentCharacterToken.chars += ch;449                return;450            }451            else {452                this.currentLocation = this.getCurrentLocation(0);453                this._emitCurrentCharacterToken(this.currentLocation);454                this.preprocessor.dropParsedChunk();455            }456        }457        this._createCharacterToken(type, ch);458    }459    _emitCodePoint(cp) {460        const type = isWhitespace(cp)461            ? token_js_1.TokenType.WHITESPACE_CHARACTER462            : cp === unicode_js_1.CODE_POINTS.NULL463                ? token_js_1.TokenType.NULL_CHARACTER464                : token_js_1.TokenType.CHARACTER;465        this._appendCharToCurrentCharacterToken(type, String.fromCodePoint(cp));466    }467    //NOTE: used when we emit characters explicitly.468    //This is always for non-whitespace and non-null characters, which allows us to avoid additional checks.469    _emitChars(ch) {470        this._appendCharToCurrentCharacterToken(token_js_1.TokenType.CHARACTER, ch);471    }472    // Character reference helpers473    _startCharacterReference() {474        this.returnState = this.state;475        this.state = State.CHARACTER_REFERENCE;476        this.entityStartPos = this.preprocessor.pos;477        this.entityDecoder.startEntity(this._isCharacterReferenceInAttribute() ? decode_1.DecodingMode.Attribute : decode_1.DecodingMode.Legacy);478    }479    _isCharacterReferenceInAttribute() {480        return (this.returnState === State.ATTRIBUTE_VALUE_DOUBLE_QUOTED ||481            this.returnState === State.ATTRIBUTE_VALUE_SINGLE_QUOTED ||482            this.returnState === State.ATTRIBUTE_VALUE_UNQUOTED);483    }484    _flushCodePointConsumedAsCharacterReference(cp) {485        if (this._isCharacterReferenceInAttribute()) {486            this.currentAttr.value += String.fromCodePoint(cp);487        }488        else {489            this._emitCodePoint(cp);490        }491    }492    // Calling states this way turns out to be much faster than any other approach.493    _callState(cp) {494        switch (this.state) {495            case State.DATA: {496                this._stateData(cp);497                break;498            }499            case State.RCDATA: {500                this._stateRcdata(cp);501                break;502            }503            case State.RAWTEXT: {504                this._stateRawtext(cp);505                break;506            }507            case State.SCRIPT_DATA: {508                this._stateScriptData(cp);509                break;510            }511            case State.PLAINTEXT: {512                this._statePlaintext(cp);513                break;514            }515            case State.TAG_OPEN: {516                this._stateTagOpen(cp);517                break;518            }519            case State.END_TAG_OPEN: {520                this._stateEndTagOpen(cp);521                break;522            }523            case State.TAG_NAME: {524                this._stateTagName(cp);525                break;526            }527            case State.RCDATA_LESS_THAN_SIGN: {528                this._stateRcdataLessThanSign(cp);529                break;530            }531            case State.RCDATA_END_TAG_OPEN: {532                this._stateRcdataEndTagOpen(cp);533                break;534            }535            case State.RCDATA_END_TAG_NAME: {536                this._stateRcdataEndTagName(cp);537                break;538            }539            case State.RAWTEXT_LESS_THAN_SIGN: {540                this._stateRawtextLessThanSign(cp);541                break;542            }543            case State.RAWTEXT_END_TAG_OPEN: {544                this._stateRawtextEndTagOpen(cp);545                break;546            }547            case State.RAWTEXT_END_TAG_NAME: {548                this._stateRawtextEndTagName(cp);549                break;550            }551            case State.SCRIPT_DATA_LESS_THAN_SIGN: {552                this._stateScriptDataLessThanSign(cp);553                break;554            }555            case State.SCRIPT_DATA_END_TAG_OPEN: {556                this._stateScriptDataEndTagOpen(cp);557                break;558            }559            case State.SCRIPT_DATA_END_TAG_NAME: {560                this._stateScriptDataEndTagName(cp);561                break;562            }563            case State.SCRIPT_DATA_ESCAPE_START: {564                this._stateScriptDataEscapeStart(cp);565                break;566            }567            case State.SCRIPT_DATA_ESCAPE_START_DASH: {568                this._stateScriptDataEscapeStartDash(cp);569                break;570            }571            case State.SCRIPT_DATA_ESCAPED: {572                this._stateScriptDataEscaped(cp);573                break;574            }575            case State.SCRIPT_DATA_ESCAPED_DASH: {576                this._stateScriptDataEscapedDash(cp);577                break;578            }579            case State.SCRIPT_DATA_ESCAPED_DASH_DASH: {580                this._stateScriptDataEscapedDashDash(cp);581                break;582            }583            case State.SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN: {584                this._stateScriptDataEscapedLessThanSign(cp);585                break;586            }587            case State.SCRIPT_DATA_ESCAPED_END_TAG_OPEN: {588                this._stateScriptDataEscapedEndTagOpen(cp);589                break;590            }591            case State.SCRIPT_DATA_ESCAPED_END_TAG_NAME: {592                this._stateScriptDataEscapedEndTagName(cp);593                break;594            }595            case State.SCRIPT_DATA_DOUBLE_ESCAPE_START: {596                this._stateScriptDataDoubleEscapeStart(cp);597                break;598            }599            case State.SCRIPT_DATA_DOUBLE_ESCAPED: {600                this._stateScriptDataDoubleEscaped(cp);601                break;602            }603            case State.SCRIPT_DATA_DOUBLE_ESCAPED_DASH: {604                this._stateScriptDataDoubleEscapedDash(cp);605                break;606            }607            case State.SCRIPT_DATA_DOUBLE_ESCAPED_DASH_DASH: {608                this._stateScriptDataDoubleEscapedDashDash(cp);609                break;610            }611            case State.SCRIPT_DATA_DOUBLE_ESCAPED_LESS_THAN_SIGN: {612                this._stateScriptDataDoubleEscapedLessThanSign(cp);613                break;614            }615            case State.SCRIPT_DATA_DOUBLE_ESCAPE_END: {616                this._stateScriptDataDoubleEscapeEnd(cp);617                break;618            }619            case State.BEFORE_ATTRIBUTE_NAME: {620                this._stateBeforeAttributeName(cp);621                break;622            }623            case State.ATTRIBUTE_NAME: {624                this._stateAttributeName(cp);625                break;626            }627            case State.AFTER_ATTRIBUTE_NAME: {628                this._stateAfterAttributeName(cp);629                break;630            }631            case State.BEFORE_ATTRIBUTE_VALUE: {632                this._stateBeforeAttributeValue(cp);633                break;634            }635            case State.ATTRIBUTE_VALUE_DOUBLE_QUOTED: {636                this._stateAttributeValueDoubleQuoted(cp);637                break;638            }639            case State.ATTRIBUTE_VALUE_SINGLE_QUOTED: {640                this._stateAttributeValueSingleQuoted(cp);641                break;642            }643            case State.ATTRIBUTE_VALUE_UNQUOTED: {644                this._stateAttributeValueUnquoted(cp);645                break;646            }647            case State.AFTER_ATTRIBUTE_VALUE_QUOTED: {648                this._stateAfterAttributeValueQuoted(cp);649                break;650            }651            case State.SELF_CLOSING_START_TAG: {652                this._stateSelfClosingStartTag(cp);653                break;654            }655            case State.BOGUS_COMMENT: {656                this._stateBogusComment(cp);657                break;658            }659            case State.MARKUP_DECLARATION_OPEN: {660                this._stateMarkupDeclarationOpen(cp);661                break;662            }663            case State.COMMENT_START: {664                this._stateCommentStart(cp);665                break;666            }667            case State.COMMENT_START_DASH: {668                this._stateCommentStartDash(cp);669                break;670            }671            case State.COMMENT: {672                this._stateComment(cp);673                break;674            }675            case State.COMMENT_LESS_THAN_SIGN: {676                this._stateCommentLessThanSign(cp);677                break;678            }679            case State.COMMENT_LESS_THAN_SIGN_BANG: {680                this._stateCommentLessThanSignBang(cp);681                break;682            }683            case State.COMMENT_LESS_THAN_SIGN_BANG_DASH: {684                this._stateCommentLessThanSignBangDash(cp);685                break;686            }687            case State.COMMENT_LESS_THAN_SIGN_BANG_DASH_DASH: {688                this._stateCommentLessThanSignBangDashDash(cp);689                break;690            }691            case State.COMMENT_END_DASH: {692                this._stateCommentEndDash(cp);693                break;694            }695            case State.COMMENT_END: {696                this._stateCommentEnd(cp);697                break;698            }699            case State.COMMENT_END_BANG: {700                this._stateCommentEndBang(cp);701                break;702            }703            case State.DOCTYPE: {704                this._stateDoctype(cp);705                break;706            }707            case State.BEFORE_DOCTYPE_NAME: {708                this._stateBeforeDoctypeName(cp);709                break;710            }711            case State.DOCTYPE_NAME: {712                this._stateDoctypeName(cp);713                break;714            }715            case State.AFTER_DOCTYPE_NAME: {716                this._stateAfterDoctypeName(cp);717                break;718            }719            case State.AFTER_DOCTYPE_PUBLIC_KEYWORD: {720                this._stateAfterDoctypePublicKeyword(cp);721                break;722            }723            case State.BEFORE_DOCTYPE_PUBLIC_IDENTIFIER: {724                this._stateBeforeDoctypePublicIdentifier(cp);725                break;726            }727            case State.DOCTYPE_PUBLIC_IDENTIFIER_DOUBLE_QUOTED: {728                this._stateDoctypePublicIdentifierDoubleQuoted(cp);729                break;730            }731            case State.DOCTYPE_PUBLIC_IDENTIFIER_SINGLE_QUOTED: {732                this._stateDoctypePublicIdentifierSingleQuoted(cp);733                break;734            }735            case State.AFTER_DOCTYPE_PUBLIC_IDENTIFIER: {736                this._stateAfterDoctypePublicIdentifier(cp);737                break;738            }739            case State.BETWEEN_DOCTYPE_PUBLIC_AND_SYSTEM_IDENTIFIERS: {740                this._stateBetweenDoctypePublicAndSystemIdentifiers(cp);741                break;742            }743            case State.AFTER_DOCTYPE_SYSTEM_KEYWORD: {744                this._stateAfterDoctypeSystemKeyword(cp);745                break;746            }747            case State.BEFORE_DOCTYPE_SYSTEM_IDENTIFIER: {748                this._stateBeforeDoctypeSystemIdentifier(cp);749                break;750            }751            case State.DOCTYPE_SYSTEM_IDENTIFIER_DOUBLE_QUOTED: {752                this._stateDoctypeSystemIdentifierDoubleQuoted(cp);753                break;754            }755            case State.DOCTYPE_SYSTEM_IDENTIFIER_SINGLE_QUOTED: {756                this._stateDoctypeSystemIdentifierSingleQuoted(cp);757                break;758            }759            case State.AFTER_DOCTYPE_SYSTEM_IDENTIFIER: {760                this._stateAfterDoctypeSystemIdentifier(cp);761                break;762            }763            case State.BOGUS_DOCTYPE: {764                this._stateBogusDoctype(cp);765                break;766            }767            case State.CDATA_SECTION: {768                this._stateCdataSection(cp);769                break;770            }771            case State.CDATA_SECTION_BRACKET: {772                this._stateCdataSectionBracket(cp);773                break;774            }775            case State.CDATA_SECTION_END: {776                this._stateCdataSectionEnd(cp);777                break;778            }779            case State.CHARACTER_REFERENCE: {780                this._stateCharacterReference();781                break;782            }783            case State.AMBIGUOUS_AMPERSAND: {784                this._stateAmbiguousAmpersand(cp);785                break;786            }787            default: {788                throw new Error('Unknown state');789            }790        }791    }792    // State machine793    // Data state794    //------------------------------------------------------------------795    _stateData(cp) {796        switch (cp) {797            case unicode_js_1.CODE_POINTS.LESS_THAN_SIGN: {798                this.state = State.TAG_OPEN;799                break;800            }801            case unicode_js_1.CODE_POINTS.AMPERSAND: {802                this._startCharacterReference();803                break;804            }805            case unicode_js_1.CODE_POINTS.NULL: {806                this._err(error_codes_js_1.ERR.unexpectedNullCharacter);807                this._emitCodePoint(cp);808                break;809            }810            case unicode_js_1.CODE_POINTS.EOF: {811                this._emitEOFToken();812                break;813            }814            default: {815                this._emitCodePoint(cp);816            }817        }818    }819    //  RCDATA state820    //------------------------------------------------------------------821    _stateRcdata(cp) {822        switch (cp) {823            case unicode_js_1.CODE_POINTS.AMPERSAND: {824                this._startCharacterReference();825                break;826            }827            case unicode_js_1.CODE_POINTS.LESS_THAN_SIGN: {828                this.state = State.RCDATA_LESS_THAN_SIGN;829                break;830            }831            case unicode_js_1.CODE_POINTS.NULL: {832                this._err(error_codes_js_1.ERR.unexpectedNullCharacter);833                this._emitChars(unicode_js_1.REPLACEMENT_CHARACTER);834                break;835            }836            case unicode_js_1.CODE_POINTS.EOF: {837                this._emitEOFToken();838                break;839            }840            default: {841                this._emitCodePoint(cp);842            }843        }844    }845    // RAWTEXT state846    //------------------------------------------------------------------847    _stateRawtext(cp) {848        switch (cp) {849            case unicode_js_1.CODE_POINTS.LESS_THAN_SIGN: {850                this.state = State.RAWTEXT_LESS_THAN_SIGN;851                break;852            }853            case unicode_js_1.CODE_POINTS.NULL: {854                this._err(error_codes_js_1.ERR.unexpectedNullCharacter);855                this._emitChars(unicode_js_1.REPLACEMENT_CHARACTER);856                break;857            }858            case unicode_js_1.CODE_POINTS.EOF: {859                this._emitEOFToken();860                break;861            }862            default: {863                this._emitCodePoint(cp);864            }865        }866    }867    // Script data state868    //------------------------------------------------------------------869    _stateScriptData(cp) {870        switch (cp) {871            case unicode_js_1.CODE_POINTS.LESS_THAN_SIGN: {872                this.state = State.SCRIPT_DATA_LESS_THAN_SIGN;873                break;874            }875            case unicode_js_1.CODE_POINTS.NULL: {876                this._err(error_codes_js_1.ERR.unexpectedNullCharacter);877                this._emitChars(unicode_js_1.REPLACEMENT_CHARACTER);878                break;879            }880            case unicode_js_1.CODE_POINTS.EOF: {881                this._emitEOFToken();882                break;883            }884            default: {885                this._emitCodePoint(cp);886            }887        }888    }889    // PLAINTEXT state890    //------------------------------------------------------------------891    _statePlaintext(cp) {892        switch (cp) {893            case unicode_js_1.CODE_POINTS.NULL: {894                this._err(error_codes_js_1.ERR.unexpectedNullCharacter);895                this._emitChars(unicode_js_1.REPLACEMENT_CHARACTER);896                break;897            }898            case unicode_js_1.CODE_POINTS.EOF: {899                this._emitEOFToken();900                break;901            }902            default: {903                this._emitCodePoint(cp);904            }905        }906    }907    // Tag open state908    //------------------------------------------------------------------909    _stateTagOpen(cp) {910        if (isAsciiLetter(cp)) {911            this._createStartTagToken();912            this.state = State.TAG_NAME;913            this._stateTagName(cp);914        }915        else916            switch (cp) {917                case unicode_js_1.CODE_POINTS.EXCLAMATION_MARK: {918                    this.state = State.MARKUP_DECLARATION_OPEN;919                    break;920                }921                case unicode_js_1.CODE_POINTS.SOLIDUS: {922                    this.state = State.END_TAG_OPEN;923                    break;924                }925                case unicode_js_1.CODE_POINTS.QUESTION_MARK: {926                    this._err(error_codes_js_1.ERR.unexpectedQuestionMarkInsteadOfTagName);927                    this._createCommentToken(1);928                    this.state = State.BOGUS_COMMENT;929                    this._stateBogusComment(cp);930                    break;931                }932                case unicode_js_1.CODE_POINTS.EOF: {933                    this._err(error_codes_js_1.ERR.eofBeforeTagName);934                    this._emitChars('<');935                    this._emitEOFToken();936                    break;937                }938                default: {939                    this._err(error_codes_js_1.ERR.invalidFirstCharacterOfTagName);940                    this._emitChars('<');941                    this.state = State.DATA;942                    this._stateData(cp);943                }944            }945    }946    // End tag open state947    //------------------------------------------------------------------948    _stateEndTagOpen(cp) {949        if (isAsciiLetter(cp)) {950            this._createEndTagToken();951            this.state = State.TAG_NAME;952            this._stateTagName(cp);953        }954        else955            switch (cp) {956                case unicode_js_1.CODE_POINTS.GREATER_THAN_SIGN: {957                    this._err(error_codes_js_1.ERR.missingEndTagName);958                    this.state = State.DATA;959                    break;960                }961                case unicode_js_1.CODE_POINTS.EOF: {962                    this._err(error_codes_js_1.ERR.eofBeforeTagName);963                    this._emitChars('</');964                    this._emitEOFToken();965                    break;966                }967                default: {968                    this._err(error_codes_js_1.ERR.invalidFirstCharacterOfTagName);969                    this._createCommentToken(2);970                    this.state = State.BOGUS_COMMENT;971                    this._stateBogusComment(cp);972                }973            }974    }975    // Tag name state976    //------------------------------------------------------------------977    _stateTagName(cp) {978        const token = this.currentToken;979        switch (cp) {980            case unicode_js_1.CODE_POINTS.SPACE:981            case unicode_js_1.CODE_POINTS.LINE_FEED:982            case unicode_js_1.CODE_POINTS.TABULATION:983            case unicode_js_1.CODE_POINTS.FORM_FEED: {984                this.state = State.BEFORE_ATTRIBUTE_NAME;985                break;986            }987            case unicode_js_1.CODE_POINTS.SOLIDUS: {988                this.state = State.SELF_CLOSING_START_TAG;989                break;990            }991            case unicode_js_1.CODE_POINTS.GREATER_THAN_SIGN: {992                this.state = State.DATA;993                this.emitCurrentTagToken();994                break;995            }996            case unicode_js_1.CODE_POINTS.NULL: {997                this._err(error_codes_js_1.ERR.unexpectedNullCharacter);998                token.tagName += unicode_js_1.REPLACEMENT_CHARACTER;999                break;1000            }1001            case unicode_js_1.CODE_POINTS.EOF: {1002                this._err(error_codes_js_1.ERR.eofInTag);1003                this._emitEOFToken();1004                break;1005            }1006            default: {1007                token.tagName += String.fromCodePoint(isAsciiUpper(cp) ? toAsciiLower(cp) : cp);1008            }1009        }1010    }1011    // RCDATA less-than sign state1012    //------------------------------------------------------------------1013    _stateRcdataLessThanSign(cp) {1014        if (cp === unicode_js_1.CODE_POINTS.SOLIDUS) {1015            this.state = State.RCDATA_END_TAG_OPEN;1016        }1017        else {1018            this._emitChars('<');1019            this.state = State.RCDATA;1020            this._stateRcdata(cp);1021        }1022    }1023    // RCDATA end tag open state1024    //------------------------------------------------------------------1025    _stateRcdataEndTagOpen(cp) {1026        if (isAsciiLetter(cp)) {1027            this.state = State.RCDATA_END_TAG_NAME;1028            this._stateRcdataEndTagName(cp);1029        }1030        else {1031            this._emitChars('</');1032            this.state = State.RCDATA;1033            this._stateRcdata(cp);1034        }1035    }1036    handleSpecialEndTag(_cp) {1037        if (!this.preprocessor.startsWith(this.lastStartTagName, false)) {1038            return !this._ensureHibernation();1039        }1040        this._createEndTagToken();1041        const token = this.currentToken;1042        token.tagName = this.lastStartTagName;1043        const cp = this.preprocessor.peek(this.lastStartTagName.length);1044        switch (cp) {1045            case unicode_js_1.CODE_POINTS.SPACE:1046            case unicode_js_1.CODE_POINTS.LINE_FEED:1047            case unicode_js_1.CODE_POINTS.TABULATION:1048            case unicode_js_1.CODE_POINTS.FORM_FEED: {1049                this._advanceBy(this.lastStartTagName.length);1050                this.state = State.BEFORE_ATTRIBUTE_NAME;1051                return false;1052            }1053            case unicode_js_1.CODE_POINTS.SOLIDUS: {1054                this._advanceBy(this.lastStartTagName.length);1055                this.state = State.SELF_CLOSING_START_TAG;1056                return false;1057            }1058            case unicode_js_1.CODE_POINTS.GREATER_THAN_SIGN: {1059                this._advanceBy(this.lastStartTagName.length);1060                this.emitCurrentTagToken();1061                this.state = State.DATA;1062                return false;1063            }1064            default: {1065                return !this._ensureHibernation();1066            }1067        }1068    }1069    // RCDATA end tag name state1070    //------------------------------------------------------------------1071    _stateRcdataEndTagName(cp) {1072        if (this.handleSpecialEndTag(cp)) {1073            this._emitChars('</');1074            this.state = State.RCDATA;1075            this._stateRcdata(cp);1076        }1077    }1078    // RAWTEXT less-than sign state1079    //------------------------------------------------------------------1080    _stateRawtextLessThanSign(cp) {1081        if (cp === unicode_js_1.CODE_POINTS.SOLIDUS) {1082            this.state = State.RAWTEXT_END_TAG_OPEN;1083        }1084        else {1085            this._emitChars('<');1086            this.state = State.RAWTEXT;1087            this._stateRawtext(cp);1088        }1089    }1090    // RAWTEXT end tag open state1091    //------------------------------------------------------------------1092    _stateRawtextEndTagOpen(cp) {1093        if (isAsciiLetter(cp)) {1094            this.state = State.RAWTEXT_END_TAG_NAME;1095            this._stateRawtextEndTagName(cp);1096        }1097        else {1098            this._emitChars('</');1099            this.state = State.RAWTEXT;1100            this._stateRawtext(cp);1101        }1102    }1103    // RAWTEXT end tag name state1104    //------------------------------------------------------------------1105    _stateRawtextEndTagName(cp) {1106        if (this.handleSpecialEndTag(cp)) {1107            this._emitChars('</');1108            this.state = State.RAWTEXT;1109            this._stateRawtext(cp);1110        }1111    }1112    // Script data less-than sign state1113    //------------------------------------------------------------------1114    _stateScriptDataLessThanSign(cp) {1115        switch (cp) {1116            case unicode_js_1.CODE_POINTS.SOLIDUS: {1117                this.state = State.SCRIPT_DATA_END_TAG_OPEN;1118                break;1119            }1120            case unicode_js_1.CODE_POINTS.EXCLAMATION_MARK: {1121                this.state = State.SCRIPT_DATA_ESCAPE_START;1122                this._emitChars('<!');1123                break;1124            }1125            default: {1126                this._emitChars('<');1127                this.state = State.SCRIPT_DATA;1128                this._stateScriptData(cp);1129            }1130        }1131    }1132    // Script data end tag open state1133    //------------------------------------------------------------------1134    _stateScriptDataEndTagOpen(cp) {1135        if (isAsciiLetter(cp)) {1136            this.state = State.SCRIPT_DATA_END_TAG_NAME;1137            this._stateScriptDataEndTagName(cp);1138        }1139        else {1140            this._emitChars('</');1141            this.state = State.SCRIPT_DATA;1142            this._stateScriptData(cp);1143        }1144    }1145    // Script data end tag name state1146    //------------------------------------------------------------------1147    _stateScriptDataEndTagName(cp) {1148        if (this.handleSpecialEndTag(cp)) {1149            this._emitChars('</');1150            this.state = State.SCRIPT_DATA;1151            this._stateScriptData(cp);1152        }1153    }1154    // Script data escape start state1155    //------------------------------------------------------------------1156    _stateScriptDataEscapeStart(cp) {1157        if (cp === unicode_js_1.CODE_POINTS.HYPHEN_MINUS) {1158            this.state = State.SCRIPT_DATA_ESCAPE_START_DASH;1159            this._emitChars('-');1160        }1161        else {1162            this.state = State.SCRIPT_DATA;1163            this._stateScriptData(cp);1164        }1165    }1166    // Script data escape start dash state1167    //------------------------------------------------------------------1168    _stateScriptDataEscapeStartDash(cp) {1169        if (cp === unicode_js_1.CODE_POINTS.HYPHEN_MINUS) {1170            this.state = State.SCRIPT_DATA_ESCAPED_DASH_DASH;1171            this._emitChars('-');1172        }1173        else {1174            this.state = State.SCRIPT_DATA;1175            this._stateScriptData(cp);1176        }1177    }1178    // Script data escaped state1179    //------------------------------------------------------------------1180    _stateScriptDataEscaped(cp) {1181        switch (cp) {1182            case unicode_js_1.CODE_POINTS.HYPHEN_MINUS: {1183                this.state = State.SCRIPT_DATA_ESCAPED_DASH;1184                this._emitChars('-');1185                break;1186            }1187            case unicode_js_1.CODE_POINTS.LESS_THAN_SIGN: {1188                this.state = State.SCRIPT_DATA_ESCAPED_LESS_THAN_SIGN;1189                break;1190            }1191            case unicode_js_1.CODE_POINTS.NULL: {1192                this._err(error_codes_js_1.ERR.unexpectedNullCharacter);1193                this._emitChars(unicode_js_1.REPLACEMENT_CHARACTER);1194                break;1195            }1196            case unicode_js_1.CODE_POINTS.EOF: {1197                this._err(error_codes_js_1.ERR.eofInScriptHtmlCommentLikeText);1198                this._emitEOFToken();1199                break;1200            }

Showing the first 1,200 of 2715 lines. Download the file for the rest.