Team Ai
Datasetpublic

Brunobkr/llama.cpp_AlgMor24_github

ΩFFFΣLLIa • llama.cpp • AlgMor24 ██████╗ ███████╗███████╗███████╗██╗ ██╗ ██╗ █████╗ ██╔═══██╗██╔════╝██╔════╝██╔════╝██║ ██║ ██║██╔══██╗ ██║ ██║█████╗ █████╗ █████╗ ██║ ██║ ██║███████║ ██║ ██║██╔══╝ ██╔══╝ ██╔══╝ ██║ ██║ ██║██╔══██║ ╚██████╔╝██║ ██║ ███████╗███████╗███████╗██║██║ ██║ ╚═════╝ ╚═╝ ╚═╝ ╚══════╝╚══════╝╚══════╝╚═╝╚═╝ ╚═╝ High-Performance LLM / VLM Inference & Autonomous Agentic Ecosystem… See the full description on the dataset page: https://huggingface.co/datasets/Brunobkr/llama.cpp_AlgMor24_github.

sourceHugging Faceupdated 2mo agoView on Hugging Face
0likes3.1kdownloads
preprocessor.d.ts37 linesDownload Raw Back to tokenizer
1import { ERR, type ParserError, type ParserErrorHandler } from '../common/error-codes.js';2export declare class Preprocessor {3    private handler;4    html: string;5    pos: number;6    private lastGapPos;7    private gapStack;8    private skipNextNewLine;9    lastChunkWritten: boolean;10    endOfChunkHit: boolean;11    bufferWaterline: number;12    private isEol;13    private lineStartPos;14    droppedBufferSize: number;15    line: number;16    constructor(handler: {17        onParseError?: ParserErrorHandler | null;18    });19    /** The column on the current line. If we just saw a gap (eg. a surrogate pair), return the index before. */20    get col(): number;21    get offset(): number;22    getError(code: ERR, cpOffset: number): ParserError;23    private lastErrOffset;24    private _err;25    private _addGap;26    private _processSurrogate;27    willDropParsedChunk(): boolean;28    dropParsedChunk(): void;29    write(chunk: string, isLastChunk: boolean): void;30    insertHtmlAtCurrentPos(chunk: string): void;31    startsWith(pattern: string, caseSensitive: boolean): boolean;32    peek(offset: number): number;33    advance(): number;34    private _checkForProblematicCharacters;35    retreat(count: number): void;36}37