Brunobkr/llama.cpp_AlgMor24_github

1/**2 * @import {Construct, State, TokenizeContext, Tokenizer} from 'micromark-util-types'3 */4 5import { factorySpace } from 'micromark-factory-space';6import { markdownLineEnding } from 'micromark-util-character';7/** @type {Construct} */8export const mathFlow = {9  tokenize: tokenizeMathFenced,10  concrete: true,11  name: 'mathFlow'12};13 14/** @type {Construct} */15const nonLazyContinuation = {16  tokenize: tokenizeNonLazyContinuation,17  partial: true18};19 20/**21 * @this {TokenizeContext}22 * @type {Tokenizer}23 */24function tokenizeMathFenced(effects, ok, nok) {25  const self = this;26  const tail = self.events[self.events.length - 1];27  const initialSize = tail && tail[1].type === "linePrefix" ? tail[2].sliceSerialize(tail[1], true).length : 0;28  let sizeOpen = 0;29  return start;30 31  /**32   * Start of math.33   *34   * ```markdown35   * > | $$36   *     ^37   *   | \frac{1}{2}38   *   | $$39   * ```40   *41   * @type {State}42   */43  function start(code) {44    effects.enter('mathFlow');45    effects.enter('mathFlowFence');46    effects.enter('mathFlowFenceSequence');47    return sequenceOpen(code);48  }49 50  /**51   * In opening fence sequence.52   *53   * ```markdown54   * > | $$55   *      ^56   *   | \frac{1}{2}57   *   | $$58   * ```59   *60   * @type {State}61   */62  function sequenceOpen(code) {63    if (code === 36) {64      effects.consume(code);65      sizeOpen++;66      return sequenceOpen;67    }68    if (sizeOpen < 2) {69      return nok(code);70    }71    effects.exit('mathFlowFenceSequence');72    return factorySpace(effects, metaBefore, "whitespace")(code);73  }74 75  /**76   * In opening fence, before meta.77   *78   * ```markdown79   * > | $$asciimath80   *       ^81   *   | x < y82   *   | $$83   * ```84   *85   * @type {State}86   */87 88  function metaBefore(code) {89    if (code === null || markdownLineEnding(code)) {90      return metaAfter(code);91    }92    effects.enter('mathFlowFenceMeta');93    effects.enter("chunkString", {94      contentType: "string"95    });96    return meta(code);97  }98 99  /**100   * In meta.101   *102   * ```markdown103   * > | $$asciimath104   *        ^105   *   | x < y106   *   | $$107   * ```108   *109   * @type {State}110   */111  function meta(code) {112    if (code === null || markdownLineEnding(code)) {113      effects.exit("chunkString");114      effects.exit('mathFlowFenceMeta');115      return metaAfter(code);116    }117    if (code === 36) {118      return nok(code);119    }120    effects.consume(code);121    return meta;122  }123 124  /**125   * After meta.126   *127   * ```markdown128   * > | $$129   *       ^130   *   | \frac{1}{2}131   *   | $$132   * ```133   *134   * @type {State}135   */136  function metaAfter(code) {137    // Guaranteed to be eol/eof.138    effects.exit('mathFlowFence');139    if (self.interrupt) {140      return ok(code);141    }142    return effects.attempt(nonLazyContinuation, beforeNonLazyContinuation, after)(code);143  }144 145  /**146   * After eol/eof in math, at a non-lazy closing fence or content.147   *148   * ```markdown149   *   | $$150   * > | \frac{1}{2}151   *     ^152   * > | $$153   *     ^154   * ```155   *156   * @type {State}157   */158  function beforeNonLazyContinuation(code) {159    return effects.attempt({160      tokenize: tokenizeClosingFence,161      partial: true162    }, after, contentStart)(code);163  }164 165  /**166   * Before math content, definitely not before a closing fence.167   *168   * ```markdown169   *   | $$170   * > | \frac{1}{2}171   *     ^172   *   | $$173   * ```174   *175   * @type {State}176   */177  function contentStart(code) {178    return (initialSize ? factorySpace(effects, beforeContentChunk, "linePrefix", initialSize + 1) : beforeContentChunk)(code);179  }180 181  /**182   * Before math content, after optional prefix.183   *184   * ```markdown185   *   | $$186   * > | \frac{1}{2}187   *     ^188   *   | $$189   * ```190   *191   * @type {State}192   */193  function beforeContentChunk(code) {194    if (code === null) {195      return after(code);196    }197    if (markdownLineEnding(code)) {198      return effects.attempt(nonLazyContinuation, beforeNonLazyContinuation, after)(code);199    }200    effects.enter('mathFlowValue');201    return contentChunk(code);202  }203 204  /**205   * In math content.206   *207   * ```markdown208   *   | $$209   * > | \frac{1}{2}210   *      ^211   *   | $$212   * ```213   *214   * @type {State}215   */216  function contentChunk(code) {217    if (code === null || markdownLineEnding(code)) {218      effects.exit('mathFlowValue');219      return beforeContentChunk(code);220    }221    effects.consume(code);222    return contentChunk;223  }224 225  /**226   * After math (ha!).227   *228   * ```markdown229   *   | $$230   *   | \frac{1}{2}231   * > | $$232   *       ^233   * ```234   *235   * @type {State}236   */237  function after(code) {238    effects.exit('mathFlow');239    return ok(code);240  }241 242  /** @type {Tokenizer} */243  function tokenizeClosingFence(effects, ok, nok) {244    let size = 0;245    /**246     * Before closing fence, at optional whitespace.247     *248     * ```markdown249     *   | $$250     *   | \frac{1}{2}251     * > | $$252     *     ^253     * ```254     */255    return factorySpace(effects, beforeSequenceClose, "linePrefix", self.parser.constructs.disable.null.includes('codeIndented') ? undefined : 4);256 257    /**258     * In closing fence, after optional whitespace, at sequence.259     *260     * ```markdown261     *   | $$262     *   | \frac{1}{2}263     * > | $$264     *     ^265     * ```266     *267     * @type {State}268     */269    function beforeSequenceClose(code) {270      effects.enter('mathFlowFence');271      effects.enter('mathFlowFenceSequence');272      return sequenceClose(code);273    }274 275    /**276     * In closing fence sequence.277     *278     * ```markdown279     *   | $$280     *   | \frac{1}{2}281     * > | $$282     *      ^283     * ```284     *285     * @type {State}286     */287    function sequenceClose(code) {288      if (code === 36) {289        size++;290        effects.consume(code);291        return sequenceClose;292      }293      if (size < sizeOpen) {294        return nok(code);295      }296      effects.exit('mathFlowFenceSequence');297      return factorySpace(effects, afterSequenceClose, "whitespace")(code);298    }299 300    /**301     * After closing fence sequence, after optional whitespace.302     *303     * ```markdown304     *   | $$305     *   | \frac{1}{2}306     * > | $$307     *       ^308     * ```309     *310     * @type {State}311     */312    function afterSequenceClose(code) {313      if (code === null || markdownLineEnding(code)) {314        effects.exit('mathFlowFence');315        return ok(code);316      }317      return nok(code);318    }319  }320}321 322/**323 * @this {TokenizeContext}324 * @type {Tokenizer}325 */326function tokenizeNonLazyContinuation(effects, ok, nok) {327  const self = this;328  return start;329 330  /** @type {State} */331  function start(code) {332    if (code === null) {333      return ok(code);334    }335    effects.enter("lineEnding");336    effects.consume(code);337    effects.exit("lineEnding");338    return lineStart;339  }340 341  /** @type {State} */342  function lineStart(code) {343    return self.parser.lazy[self.now().line] ? nok(code) : ok(code);344  }345}