Brunobkr/llama.cpp_AlgMor24_github

1/**2 * This file contains the “gullet” where macros are expanded3 * until only non-macro tokens remain.4 */5 6import functions from "./functions";7import symbols from "./symbols";8import Lexer from "./Lexer";9import {Token} from "./Token";10import type {Mode} from "./types";11import ParseError from "./ParseError";12import SourceLocation from "./SourceLocation";13import Namespace from "./Namespace";14import macros from "./macros";15 16import type {MacroContextInterface, MacroDefinition, MacroExpansion, MacroArg}17    from "./defineMacro";18import type Settings from "./Settings";19 20// List of commands that act like macros but aren't defined as a macro,21// function, or symbol.  Used in `isDefined`.22export const implicitCommands = {23    "^": true,           // Parser.js24    "_": true,           // Parser.js25    "\\limits": true,    // Parser.js26    "\\nolimits": true,  // Parser.js27};28 29export default class MacroExpander implements MacroContextInterface {30    settings: Settings;31    expansionCount: number;32    lexer!: Lexer;33    macros: Namespace<MacroDefinition>;34    stack: Token[];35    mode: Mode;36 37    constructor(input: string, settings: Settings, mode: Mode) {38        this.settings = settings;39        this.expansionCount = 0;40        this.feed(input);41        // Make new global namespace42        this.macros = new Namespace(macros, settings.macros);43        this.mode = mode;44        this.stack = []; // contains tokens in REVERSE order45    }46 47    /**48     * Feed a new input string to the same MacroExpander49     * (with existing macros etc.).50     */51    feed(input: string) {52        this.lexer = new Lexer(input, this.settings);53    }54 55    /**56     * Switches between "text" and "math" modes.57     */58    switchMode(newMode: Mode) {59        this.mode = newMode;60    }61 62    /**63     * Start a new group nesting within all namespaces.64     */65    beginGroup() {66        this.macros.beginGroup();67    }68 69    /**70     * End current group nesting within all namespaces.71     */72    endGroup() {73        this.macros.endGroup();74    }75 76    /**77     * Ends all currently nested groups (if any), restoring values before the78     * groups began.  Useful in case of an error in the middle of parsing.79     */80    endGroups() {81        this.macros.endGroups();82    }83 84    /**85     * Returns the topmost token on the stack, without expanding it.86     * Similar in behavior to TeX's `\futurelet`.87     */88    future(): Token {89        if (this.stack.length === 0) {90            this.pushToken(this.lexer.lex());91        }92        return this.stack[this.stack.length - 1];93    }94 95    /**96     * Remove and return the next unexpanded token.97     */98    popToken(): Token {99        this.future();  // ensure non-empty stack100        return this.stack.pop()!;101    }102 103    /**104     * Add a given token to the token stack.  In particular, this get be used105     * to put back a token returned from one of the other methods.106     */107    pushToken(token: Token) {108        this.stack.push(token);109    }110 111    /**112     * Append an array of tokens to the token stack.113     */114    pushTokens(tokens: Token[]) {115        this.stack.push(...tokens);116    }117 118    /**119     * Find an macro argument without expanding tokens and append the array of120     * tokens to the token stack. Uses Token as a container for the result.121     */122    scanArgument(isOptional: boolean): Token | null | undefined {123        let start;124        let end;125        let tokens;126        if (isOptional) {127            this.consumeSpaces(); // \@ifnextchar gobbles any space following it128            if (this.future().text !== "[") {129                return null;130            }131            start = this.popToken(); // don't include [ in tokens132            ({tokens, end} = this.consumeArg(["]"]));133        } else {134            ({tokens, start, end} = this.consumeArg());135        }136 137        // indicate the end of an argument138        this.pushToken(new Token("EOF", end.loc));139 140        this.pushTokens(tokens);141        return new Token("", SourceLocation.range(start, end));142    }143 144    /**145     * Consume all following space tokens, without expansion.146     */147    consumeSpaces() {148        for (;;) {149            const token = this.future();150            if (token.text === " ") {151                this.stack.pop();152            } else {153                break;154            }155        }156    }157 158    /**159     * Consume an argument from the token stream, and return the resulting array160     * of tokens and start/end token.161     */162    consumeArg(delims?: string[] | null | undefined): MacroArg {163        // The argument for a delimited parameter is the shortest (possibly164        // empty) sequence of tokens with properly nested {...} groups that is165        // followed ... by this particular list of non-parameter tokens.166        // The argument for an undelimited parameter is the next nonblank167        // token, unless that token is ‘{’, when the argument will be the168        // entire {...} group that follows.169        const tokens: Token[] = [];170        const isDelimited = delims && delims.length > 0;171        if (!isDelimited) {172            // Ignore spaces between arguments.  As the TeXbook says:173            // "After you have said ‘\def\row#1#2{...}’, you are allowed to174            //  put spaces between the arguments (e.g., ‘\row x n’), because175            //  TeX doesn’t use single spaces as undelimited arguments."176            this.consumeSpaces();177        }178        const start = this.future();179        let tok;180        let depth = 0;181        let match = 0;182        do {183            tok = this.popToken();184            tokens.push(tok);185            if (tok.text === "{") {186                ++depth;187            } else if (tok.text === "}") {188                --depth;189                if (depth === -1) {190                    throw new ParseError("Extra }", tok);191                }192            } else if (tok.text === "EOF") {193                throw new ParseError("Unexpected end of input in a macro argument" +194                    ", expected '" + (delims && isDelimited ? delims[match] : "}") +195                    "'", tok);196            }197            if (delims && isDelimited) {198                if ((depth === 0 || (depth === 1 && delims[match] === "{")) &&199                    tok.text === delims[match]) {200                    ++match;201                    if (match === delims.length) {202                        // don't include delims in tokens203                        tokens.splice(-match, match);204                        break;205                    }206                } else {207                    match = 0;208                }209            }210        } while (depth !== 0 || isDelimited);211        // If the argument found ... has the form ‘{<nested tokens>}’,212        // ... the outermost braces enclosing the argument are removed213        if (start.text === "{" && tokens[tokens.length - 1].text === "}") {214            tokens.pop();215            tokens.shift();216        }217        tokens.reverse(); // to fit in with stack order218        return {tokens, start, end: tok};219    }220 221    /**222     * Consume the specified number of (delimited) arguments from the token223     * stream and return the resulting array of arguments.224     */225    consumeArgs(numArgs: number, delimiters?: string[][]): Token[][] {226        if (delimiters) {227            if (delimiters.length !== numArgs + 1) {228                throw new ParseError(229                    "The length of delimiters doesn't match the number of args!");230            }231            const delims = delimiters[0];232            for (let i = 0; i < delims.length; i++) {233                const tok = this.popToken();234                if (delims[i] !== tok.text) {235                    throw new ParseError(236                        "Use of the macro doesn't match its definition", tok);237                }238            }239        }240 241        const args: Token[][] = [];242        for (let i = 0; i < numArgs; i++) {243            args.push(this.consumeArg(delimiters && delimiters[i + 1]).tokens);244        }245        return args;246    }247 248    /**249     * Increment `expansionCount` by the specified amount.250     * Throw an error if it exceeds `maxExpand`.251     */252    countExpansion(amount: number): void {253        this.expansionCount += amount;254        if (this.expansionCount > this.settings.maxExpand) {255            throw new ParseError("Too many expansions: infinite loop or " +256                "need to increase maxExpand setting");257        }258    }259 260    /**261     * Expand the next token only once if possible.262     *263     * If the token is expanded, the resulting tokens will be pushed onto264     * the stack in reverse order, and the number of such tokens will be265     * returned.  This number might be zero or positive.266     *267     * If not, the return value is `false`, and the next token remains at the268     * top of the stack.269     *270     * In either case, the next token will be on the top of the stack,271     * or the stack will be empty (in case of empty expansion272     * and no other tokens).273     *274     * Used to implement `expandAfterFuture` and `expandNextToken`.275     *276     * If expandableOnly, only expandable tokens are expanded and277     * an undefined control sequence results in an error.278     */279    expandOnce(expandableOnly?: boolean): number | boolean {280        const topToken = this.popToken();281        const name = topToken.text;282        const expansion = !topToken.noexpand ? this._getExpansion(name) : null;283        if (expansion == null || (expandableOnly && expansion.unexpandable)) {284            if (expandableOnly && expansion == null &&285                    name[0] === "\\" && !this.isDefined(name)) {286                throw new ParseError("Undefined control sequence: " + name);287            }288            this.pushToken(topToken);289            return false;290        }291        this.countExpansion(1);292        let tokens = expansion.tokens;293        const args = this.consumeArgs(expansion.numArgs, expansion.delimiters);294        if (expansion.numArgs) {295            // paste arguments in place of the placeholders296            tokens = tokens.slice(); // make a shallow copy297            for (let i = tokens.length - 1; i >= 0; --i) {298                let tok = tokens[i];299                if (tok.text === "#") {300                    if (i === 0) {301                        throw new ParseError(302                            "Incomplete placeholder at end of macro body",303                            tok);304                    }305                    tok = tokens[--i]; // next token on stack306                    if (tok.text === "#") { // ## → #307                        tokens.splice(i + 1, 1); // drop first #308                    } else if (/^[1-9]$/.test(tok.text)) {309                        // replace the placeholder with the indicated argument310                        tokens.splice(i, 2, ...args[+tok.text - 1]);311                    } else {312                        throw new ParseError(313                            "Not a valid argument number",314                            tok);315                    }316                }317            }318        }319        // Concatenate expansion onto top of stack.320        this.pushTokens(tokens);321        return tokens.length;322    }323 324    /**325     * Expand the next token only once (if possible), and return the resulting326     * top token on the stack (without removing anything from the stack).327     * Similar in behavior to TeX's `\expandafter\futurelet`.328     * Equivalent to expandOnce() followed by future().329     */330    expandAfterFuture(): Token {331        this.expandOnce();332        return this.future();333    }334 335    /**336     * Recursively expand first token, then return first non-expandable token.337     */338    expandNextToken(): Token {339        for (;;) {340            if (this.expandOnce() === false) {  // fully expanded341                const token = this.stack.pop()!;342                // the token after \noexpand is interpreted as if its meaning343                // were ‘\relax’344                if (token.treatAsRelax) {345                    token.text = "\\relax";346                }347                return token;348            }349        }350    }351 352    /**353     * Fully expand the given macro name and return the resulting list of354     * tokens, or return `undefined` if no such macro is defined.355     */356    expandMacro(name: string): Token[] | undefined {357        return this.macros.has(name)358            ? this.expandTokens([new Token(name)]) : undefined;359    }360 361    /**362     * Fully expand the given token stream and return the resulting list of363     * tokens.  Note that the input tokens are in reverse order, but the364     * output tokens are in forward order.365     */366    expandTokens(tokens: Token[]): Token[] {367        const output: Token[] = [];368        const oldStackLength = this.stack.length;369        this.pushTokens(tokens);370        while (this.stack.length > oldStackLength) {371            // Expand only expandable tokens372            if (this.expandOnce(true) === false) {  // fully expanded373                const token = this.stack.pop()!;374                if (token.treatAsRelax) {375                    // the expansion of \noexpand is the token itself376                    token.noexpand = false;377                    token.treatAsRelax = false;378                }379                output.push(token);380            }381        }382        // Count all of these tokens as additional expansions, to prevent383        // exponential blowup from linearly many \edef's.384        this.countExpansion(output.length);385        return output;386    }387 388    /**389     * Fully expand the given macro name and return the result as a string,390     * or return `undefined` if no such macro is defined.391     */392    expandMacroAsText(name: string): string | undefined {393        const tokens = this.expandMacro(name);394        if (tokens) {395            return tokens.map((token) => token.text).join("");396        } else {397            return tokens;398        }399    }400 401    /**402     * Returns the expanded macro as a reversed array of tokens and a macro403     * argument count.  Or returns `null` if no such macro.404     */405    _getExpansion(name: string): MacroExpansion | null | undefined {406        const definition = this.macros.get(name);407 408        if (definition == null) { // mainly checking for undefined here409            return definition;410        }411        // If a single character has an associated catcode other than 13412        // (active character), then don't expand it.413        if (name.length === 1) {414            const catcode = this.lexer.catcodes[name];415            if (catcode != null && catcode !== 13) {416                return;417            }418        }419        const expansion =420            typeof definition === "function" ? definition(this) : definition;421        if (typeof expansion === "string") {422            let numArgs = 0;423            if (expansion.includes("#")) {424                const stripped = expansion.replace(/##/g, "");425                while (stripped.includes("#" + (numArgs + 1))) {426                    ++numArgs;427                }428            }429            const bodyLexer = new Lexer(expansion, this.settings);430            const tokens = [];431            let tok = bodyLexer.lex();432            while (tok.text !== "EOF") {433                tokens.push(tok);434                tok = bodyLexer.lex();435            }436            tokens.reverse(); // to fit in with stack using push and pop437            const expanded = {tokens, numArgs};438            return expanded;439        }440 441        return expansion;442    }443 444    /**445     * Determine whether a command is currently "defined" (has some446     * functionality), meaning that it's a macro (in the current group),447     * a function, a symbol, or one of the special commands listed in448     * `implicitCommands`.449     */450    isDefined(name: string): boolean {451        return this.macros.has(name) ||452            functions.hasOwnProperty(name) ||453            symbols.math.hasOwnProperty(name) ||454            symbols.text.hasOwnProperty(name) ||455            implicitCommands.hasOwnProperty(name);456    }457 458    /**459     * Determine whether a command is expandable.460     */461    isExpandable(name: string): boolean {462        const macro = this.macros.get(name);463        return macro != null ? typeof macro === "string"464                || typeof macro === "function" || !macro.unexpandable465            : functions.hasOwnProperty(name) && !functions[name].primitive;466    }467}468