Brunobkr/llama.cpp_AlgMor24_github

1import SourceLocation from "./SourceLocation";2 3/**4 * Interface required to break circular dependency between Token, Lexer, and5 * ParseError.6 */7export interface LexerInterface {8    input: string;9    tokenRegex: RegExp;10}11 12/**13 * The resulting token returned from `lex`.14 *15 * It consists of the token text plus some position information.16 * The position information is essentially a range in an input string,17 * but instead of referencing the bare input string, we refer to the lexer.18 * That way it is possible to attach extra metadata to the input string,19 * like for example a file name or similar.20 *21 * The position information is optional, so it is OK to construct synthetic22 * tokens if appropriate. Not providing available position information may23 * lead to degraded error reporting, though.24 */25export class Token {26    text: string;27    loc: SourceLocation | null | undefined;28    noexpand: boolean | null | undefined; // don't expand the token29    treatAsRelax: boolean | null | undefined; // used in \noexpand30 31    constructor(32        text: string,           // the text of this token33        loc?: SourceLocation | null | undefined,34    ) {35        this.text = text;36        this.loc = loc;37    }38 39    /**40     * Given a pair of tokens (this and endToken), compute a `Token` encompassing41     * the whole input range enclosed by these two.42     */43    range(44        endToken: Token,  // last token of the range, inclusive45        text: string,     // the text of the newly constructed token46    ): Token {47        return new Token(text, SourceLocation.range(this, endToken));48    }49}50