Brunobkr/llama.cpp_AlgMor24_github

1/**2 * @import {Chunk, Code, Encoding, Value} from 'micromark-util-types'3 */4 5/**6 * @callback Preprocessor7 *   Preprocess a value.8 * @param {Value} value9 *   Value.10 * @param {Encoding | null | undefined} [encoding]11 *   Encoding when `value` is a typed array (optional).12 * @param {boolean | null | undefined} [end=false]13 *   Whether this is the last chunk (default: `false`).14 * @returns {Array<Chunk>}15 *   Chunks.16 */17 18import {codes, constants} from 'micromark-util-symbol'19 20const search = /[\0\t\n\r]/g21 22/**23 * @returns {Preprocessor}24 *   Preprocess a value.25 */26export function preprocess() {27  let column = 128  let buffer = ''29  /** @type {boolean | undefined} */30  let start = true31  /** @type {boolean | undefined} */32  let atCarriageReturn33 34  return preprocessor35 36  /** @type {Preprocessor} */37  // eslint-disable-next-line complexity38  function preprocessor(value, encoding, end) {39    /** @type {Array<Chunk>} */40    const chunks = []41    /** @type {RegExpMatchArray | null} */42    let match43    /** @type {number} */44    let next45    /** @type {number} */46    let startPosition47    /** @type {number} */48    let endPosition49    /** @type {Code} */50    let code51 52    value =53      buffer +54      (typeof value === 'string'55        ? value.toString()56        : new TextDecoder(encoding || undefined).decode(value))57 58    startPosition = 059    buffer = ''60 61    if (start) {62      // To do: `markdown-rs` actually parses BOMs (byte order mark).63      if (value.charCodeAt(0) === codes.byteOrderMarker) {64        startPosition++65      }66 67      start = undefined68    }69 70    while (startPosition < value.length) {71      search.lastIndex = startPosition72      match = search.exec(value)73      endPosition =74        match && match.index !== undefined ? match.index : value.length75      code = value.charCodeAt(endPosition)76 77      if (!match) {78        buffer = value.slice(startPosition)79        break80      }81 82      if (83        code === codes.lf &&84        startPosition === endPosition &&85        atCarriageReturn86      ) {87        chunks.push(codes.carriageReturnLineFeed)88        atCarriageReturn = undefined89      } else {90        if (atCarriageReturn) {91          chunks.push(codes.carriageReturn)92          atCarriageReturn = undefined93        }94 95        if (startPosition < endPosition) {96          chunks.push(value.slice(startPosition, endPosition))97          column += endPosition - startPosition98        }99 100        switch (code) {101          case codes.nul: {102            chunks.push(codes.replacementCharacter)103            column++104 105            break106          }107 108          case codes.ht: {109            next = Math.ceil(column / constants.tabSize) * constants.tabSize110            chunks.push(codes.horizontalTab)111            while (column++ < next) chunks.push(codes.virtualSpace)112 113            break114          }115 116          case codes.lf: {117            chunks.push(codes.lineFeed)118            column = 1119 120            break121          }122 123          default: {124            atCarriageReturn = true125            column = 1126          }127        }128      }129 130      startPosition = endPosition + 1131    }132 133    if (end) {134      if (atCarriageReturn) chunks.push(codes.carriageReturn)135      if (buffer) chunks.push(buffer)136      chunks.push(codes.eof)137    }138 139    return chunks140  }141}142