Brunobkr/llama.cpp_AlgMor24_github

1/**2 * @import {Chunk, Code, Encoding, Value} from 'micromark-util-types'3 */4 5/**6 * @callback Preprocessor7 *   Preprocess a value.8 * @param {Value} value9 *   Value.10 * @param {Encoding | null | undefined} [encoding]11 *   Encoding when `value` is a typed array (optional).12 * @param {boolean | null | undefined} [end=false]13 *   Whether this is the last chunk (default: `false`).14 * @returns {Array<Chunk>}15 *   Chunks.16 */17 18const search = /[\0\t\n\r]/g;19 20/**21 * @returns {Preprocessor}22 *   Preprocess a value.23 */24export function preprocess() {25  let column = 1;26  let buffer = '';27  /** @type {boolean | undefined} */28  let start = true;29  /** @type {boolean | undefined} */30  let atCarriageReturn;31  return preprocessor;32 33  /** @type {Preprocessor} */34  // eslint-disable-next-line complexity35  function preprocessor(value, encoding, end) {36    /** @type {Array<Chunk>} */37    const chunks = [];38    /** @type {RegExpMatchArray | null} */39    let match;40    /** @type {number} */41    let next;42    /** @type {number} */43    let startPosition;44    /** @type {number} */45    let endPosition;46    /** @type {Code} */47    let code;48    value = buffer + (typeof value === 'string' ? value.toString() : new TextDecoder(encoding || undefined).decode(value));49    startPosition = 0;50    buffer = '';51    if (start) {52      // To do: `markdown-rs` actually parses BOMs (byte order mark).53      if (value.charCodeAt(0) === 65279) {54        startPosition++;55      }56      start = undefined;57    }58    while (startPosition < value.length) {59      search.lastIndex = startPosition;60      match = search.exec(value);61      endPosition = match && match.index !== undefined ? match.index : value.length;62      code = value.charCodeAt(endPosition);63      if (!match) {64        buffer = value.slice(startPosition);65        break;66      }67      if (code === 10 && startPosition === endPosition && atCarriageReturn) {68        chunks.push(-3);69        atCarriageReturn = undefined;70      } else {71        if (atCarriageReturn) {72          chunks.push(-5);73          atCarriageReturn = undefined;74        }75        if (startPosition < endPosition) {76          chunks.push(value.slice(startPosition, endPosition));77          column += endPosition - startPosition;78        }79        switch (code) {80          case 0:81            {82              chunks.push(65533);83              column++;84              break;85            }86          case 9:87            {88              next = Math.ceil(column / 4) * 4;89              chunks.push(-2);90              while (column++ < next) chunks.push(-1);91              break;92            }93          case 10:94            {95              chunks.push(-4);96              column = 1;97              break;98            }99          default:100            {101              atCarriageReturn = true;102              column = 1;103            }104        }105      }106      startPosition = endPosition + 1;107    }108    if (end) {109      if (atCarriageReturn) chunks.push(-5);110      if (buffer) chunks.push(buffer);111      chunks.push(null);112    }113    return chunks;114  }115}