Brunobkr/llama.cpp_AlgMor24_github

1/**2 * @import {Options} from 'micromark-extension-math'3 * @import {Construct, Previous, Resolver, State, Token, TokenizeContext, Tokenizer} from 'micromark-util-types'4 */5 6// To do: next major: clean spaces in HTML compiler.7// This has to be coordinated together with `mdast-util-math`.8 9import { markdownLineEnding } from 'micromark-util-character';10/**11 * @param {Options | null | undefined} [options={}]12 *   Configuration (default: `{}`).13 * @returns {Construct}14 *   Construct.15 */16export function mathText(options) {17  const options_ = options || {};18  let single = options_.singleDollarTextMath;19  if (single === null || single === undefined) {20    single = true;21  }22  return {23    tokenize: tokenizeMathText,24    resolve: resolveMathText,25    previous,26    name: 'mathText'27  };28 29  /**30   * @this {TokenizeContext}31   * @type {Tokenizer}32   */33  function tokenizeMathText(effects, ok, nok) {34    const self = this;35    let sizeOpen = 0;36    /** @type {number} */37    let size;38    /** @type {Token} */39    let token;40    return start;41 42    /**43     * Start of math (text).44     *45     * ```markdown46     * > | $a$47     *     ^48     * > | \$a$49     *      ^50     * ```51     *52     * @type {State}53     */54    function start(code) {55      effects.enter('mathText');56      effects.enter('mathTextSequence');57      return sequenceOpen(code);58    }59 60    /**61     * In opening sequence.62     *63     * ```markdown64     * > | $a$65     *     ^66     * ```67     *68     * @type {State}69     */70 71    function sequenceOpen(code) {72      if (code === 36) {73        effects.consume(code);74        sizeOpen++;75        return sequenceOpen;76      }77 78      // Not enough markers in the sequence.79      if (sizeOpen < 2 && !single) {80        return nok(code);81      }82      effects.exit('mathTextSequence');83      return between(code);84    }85 86    /**87     * Between something and something else.88     *89     * ```markdown90     * > | $a$91     *      ^^92     * ```93     *94     * @type {State}95     */96    function between(code) {97      if (code === null) {98        return nok(code);99      }100      if (code === 36) {101        token = effects.enter('mathTextSequence');102        size = 0;103        return sequenceClose(code);104      }105 106      // Tabs don’t work, and virtual spaces don’t make sense.107      if (code === 32) {108        effects.enter('space');109        effects.consume(code);110        effects.exit('space');111        return between;112      }113      if (markdownLineEnding(code)) {114        effects.enter("lineEnding");115        effects.consume(code);116        effects.exit("lineEnding");117        return between;118      }119 120      // Data.121      effects.enter('mathTextData');122      return data(code);123    }124 125    /**126     * In data.127     *128     * ```markdown129     * > | $a$130     *      ^131     * ```132     *133     * @type {State}134     */135    function data(code) {136      if (code === null || code === 32 || code === 36 || markdownLineEnding(code)) {137        effects.exit('mathTextData');138        return between(code);139      }140      effects.consume(code);141      return data;142    }143 144    /**145     * In closing sequence.146     *147     * ```markdown148     * > | `a`149     *       ^150     * ```151     *152     * @type {State}153     */154 155    function sequenceClose(code) {156      // More.157      if (code === 36) {158        effects.consume(code);159        size++;160        return sequenceClose;161      }162 163      // Done!164      if (size === sizeOpen) {165        effects.exit('mathTextSequence');166        effects.exit('mathText');167        return ok(code);168      }169 170      // More or less accents: mark as data.171      token.type = 'mathTextData';172      return data(code);173    }174  }175}176 177/** @type {Resolver} */178function resolveMathText(events) {179  let tailExitIndex = events.length - 4;180  let headEnterIndex = 3;181  /** @type {number} */182  let index;183  /** @type {number | undefined} */184  let enter;185 186  // If we start and end with an EOL or a space.187  if ((events[headEnterIndex][1].type === "lineEnding" || events[headEnterIndex][1].type === 'space') && (events[tailExitIndex][1].type === "lineEnding" || events[tailExitIndex][1].type === 'space')) {188    index = headEnterIndex;189 190    // And we have data.191    while (++index < tailExitIndex) {192      if (events[index][1].type === 'mathTextData') {193        // Then we have padding.194        events[tailExitIndex][1].type = 'mathTextPadding';195        events[headEnterIndex][1].type = 'mathTextPadding';196        headEnterIndex += 2;197        tailExitIndex -= 2;198        break;199      }200    }201  }202 203  // Merge adjacent spaces and data.204  index = headEnterIndex - 1;205  tailExitIndex++;206  while (++index <= tailExitIndex) {207    if (enter === undefined) {208      if (index !== tailExitIndex && events[index][1].type !== "lineEnding") {209        enter = index;210      }211    } else if (index === tailExitIndex || events[index][1].type === "lineEnding") {212      events[enter][1].type = 'mathTextData';213      if (index !== enter + 2) {214        events[enter][1].end = events[index - 1][1].end;215        events.splice(enter + 2, index - enter - 2);216        tailExitIndex -= index - enter - 2;217        index = enter + 2;218      }219      enter = undefined;220    }221  }222  return events;223}224 225/**226 * @this {TokenizeContext}227 * @type {Previous}228 */229function previous(code) {230  // If there is a previous code, there will always be a tail.231  return code !== 36 || this.events[this.events.length - 1][1].type === "characterEscape";232}