Brunobkr/llama.cpp_AlgMor24_github

1/**2 * @import {3 *   InitialConstruct,4 *   Initializer,5 *   State,6 *   TokenizeContext,7 *   Token8 * } from 'micromark-util-types'9 */10 11import { factorySpace } from 'micromark-factory-space';12import { markdownLineEnding } from 'micromark-util-character';13/** @type {InitialConstruct} */14export const content = {15  tokenize: initializeContent16};17 18/**19 * @this {TokenizeContext}20 *   Context.21 * @type {Initializer}22 *   Content.23 */24function initializeContent(effects) {25  const contentStart = effects.attempt(this.parser.constructs.contentInitial, afterContentStartConstruct, paragraphInitial);26  /** @type {Token} */27  let previous;28  return contentStart;29 30  /** @type {State} */31  function afterContentStartConstruct(code) {32    if (code === null) {33      effects.consume(code);34      return;35    }36    effects.enter("lineEnding");37    effects.consume(code);38    effects.exit("lineEnding");39    return factorySpace(effects, contentStart, "linePrefix");40  }41 42  /** @type {State} */43  function paragraphInitial(code) {44    effects.enter("paragraph");45    return lineStart(code);46  }47 48  /** @type {State} */49  function lineStart(code) {50    const token = effects.enter("chunkText", {51      contentType: "text",52      previous53    });54    if (previous) {55      previous.next = token;56    }57    previous = token;58    return data(code);59  }60 61  /** @type {State} */62  function data(code) {63    if (code === null) {64      effects.exit("chunkText");65      effects.exit("paragraph");66      effects.consume(code);67      return;68    }69    if (markdownLineEnding(code)) {70      effects.consume(code);71      effects.exit("chunkText");72      return lineStart;73    }74 75    // Data.76    effects.consume(code);77    return data;78  }79}