Brunobkr/llama.cpp_AlgMor24_github

1/**2 * Create a tokenizer.3 * Tokenizers deal with one type of data (e.g., containers, flow, text).4 * The parser is the object dealing with it all.5 * `initialize` works like other constructs, except that only its `tokenize`6 * function is used, in which case it doesn’t receive an `ok` or `nok`.7 * `from` can be given to set the point before the first character, although8 * when further lines are indented, they must be set with `defineSkip`.9 *10 * @param {ParseContext} parser11 *   Parser.12 * @param {InitialConstruct} initialize13 *   Construct.14 * @param {Omit<Point, '_bufferIndex' | '_index'> | undefined} [from]15 *   Point (optional).16 * @returns {TokenizeContext}17 *   Context.18 */19export function createTokenizer(parser: ParseContext, initialize: InitialConstruct, from?: Omit<Point, "_bufferIndex" | "_index"> | undefined): TokenizeContext;20/**21 * Restore the state.22 */23export type Restore = () => undefined;24/**25 * Info.26 */27export type Info = {28    /**29     *   Restore.30     */31    restore: Restore;32    /**33     *   From.34     */35    from: number;36};37/**38 * Handle a successful run.39 */40export type ReturnHandle = (construct: Construct, info: Info) => undefined;41import type { ParseContext } from 'micromark-util-types';42import type { InitialConstruct } from 'micromark-util-types';43import type { Point } from 'micromark-util-types';44import type { TokenizeContext } from 'micromark-util-types';45import type { Construct } from 'micromark-util-types';46//# sourceMappingURL=create-tokenizer.d.ts.map