Brunobkr/llama.cpp_AlgMor24_github

1/**2 * @import {3 *   Construct,4 *   ContainerState,5 *   InitialConstruct,6 *   Initializer,7 *   Point,8 *   State,9 *   TokenizeContext,10 *   Tokenizer,11 *   Token12 * } from 'micromark-util-types'13 */14 15/**16 * @typedef {[Construct, ContainerState]} StackItem17 *   Construct and its state.18 */19 20import { factorySpace } from 'micromark-factory-space';21import { markdownLineEnding } from 'micromark-util-character';22import { splice } from 'micromark-util-chunked';23/** @type {InitialConstruct} */24export const document = {25  tokenize: initializeDocument26};27 28/** @type {Construct} */29const containerConstruct = {30  tokenize: tokenizeContainer31};32 33/**34 * @this {TokenizeContext}35 *   Self.36 * @type {Initializer}37 *   Initializer.38 */39function initializeDocument(effects) {40  const self = this;41  /** @type {Array<StackItem>} */42  const stack = [];43  let continued = 0;44  /** @type {TokenizeContext | undefined} */45  let childFlow;46  /** @type {Token | undefined} */47  let childToken;48  /** @type {number} */49  let lineStartOffset;50  return start;51 52  /** @type {State} */53  function start(code) {54    // First we iterate through the open blocks, starting with the root55    // document, and descending through last children down to the last open56    // block.57    // Each block imposes a condition that the line must satisfy if the block is58    // to remain open.59    // For example, a block quote requires a `>` character.60    // A paragraph requires a non-blank line.61    // In this phase we may match all or just some of the open blocks.62    // But we cannot close unmatched blocks yet, because we may have a lazy63    // continuation line.64    if (continued < stack.length) {65      const item = stack[continued];66      self.containerState = item[1];67      return effects.attempt(item[0].continuation, documentContinue, checkNewContainers)(code);68    }69 70    // Done.71    return checkNewContainers(code);72  }73 74  /** @type {State} */75  function documentContinue(code) {76    continued++;77 78    // Note: this field is called `_closeFlow` but it also closes containers.79    // Perhaps a good idea to rename it but it’s already used in the wild by80    // extensions.81    if (self.containerState._closeFlow) {82      self.containerState._closeFlow = undefined;83      if (childFlow) {84        closeFlow();85      }86 87      // Note: this algorithm for moving events around is similar to the88      // algorithm when dealing with lazy lines in `writeToChild`.89      const indexBeforeExits = self.events.length;90      let indexBeforeFlow = indexBeforeExits;91      /** @type {Point | undefined} */92      let point;93 94      // Find the flow chunk.95      while (indexBeforeFlow--) {96        if (self.events[indexBeforeFlow][0] === 'exit' && self.events[indexBeforeFlow][1].type === "chunkFlow") {97          point = self.events[indexBeforeFlow][1].end;98          break;99        }100      }101      exitContainers(continued);102 103      // Fix positions.104      let index = indexBeforeExits;105      while (index < self.events.length) {106        self.events[index][1].end = {107          ...point108        };109        index++;110      }111 112      // Inject the exits earlier (they’re still also at the end).113      splice(self.events, indexBeforeFlow + 1, 0, self.events.slice(indexBeforeExits));114 115      // Discard the duplicate exits.116      self.events.length = index;117      return checkNewContainers(code);118    }119    return start(code);120  }121 122  /** @type {State} */123  function checkNewContainers(code) {124    // Next, after consuming the continuation markers for existing blocks, we125    // look for new block starts (e.g. `>` for a block quote).126    // If we encounter a new block start, we close any blocks unmatched in127    // step 1 before creating the new block as a child of the last matched128    // block.129    if (continued === stack.length) {130      // No need to `check` whether there’s a container, of `exitContainers`131      // would be moot.132      // We can instead immediately `attempt` to parse one.133      if (!childFlow) {134        return documentContinued(code);135      }136 137      // If we have concrete content, such as block HTML or fenced code,138      // we can’t have containers “pierce” into them, so we can immediately139      // start.140      if (childFlow.currentConstruct && childFlow.currentConstruct.concrete) {141        return flowStart(code);142      }143 144      // If we do have flow, it could still be a blank line,145      // but we’d be interrupting it w/ a new container if there’s a current146      // construct.147      // To do: next major: remove `_gfmTableDynamicInterruptHack` (no longer148      // needed in micromark-extension-gfm-table@1.0.6).149      self.interrupt = Boolean(childFlow.currentConstruct && !childFlow._gfmTableDynamicInterruptHack);150    }151 152    // Check if there is a new container.153    self.containerState = {};154    return effects.check(containerConstruct, thereIsANewContainer, thereIsNoNewContainer)(code);155  }156 157  /** @type {State} */158  function thereIsANewContainer(code) {159    if (childFlow) closeFlow();160    exitContainers(continued);161    return documentContinued(code);162  }163 164  /** @type {State} */165  function thereIsNoNewContainer(code) {166    self.parser.lazy[self.now().line] = continued !== stack.length;167    lineStartOffset = self.now().offset;168    return flowStart(code);169  }170 171  /** @type {State} */172  function documentContinued(code) {173    // Try new containers.174    self.containerState = {};175    return effects.attempt(containerConstruct, containerContinue, flowStart)(code);176  }177 178  /** @type {State} */179  function containerContinue(code) {180    continued++;181    stack.push([self.currentConstruct, self.containerState]);182    // Try another.183    return documentContinued(code);184  }185 186  /** @type {State} */187  function flowStart(code) {188    if (code === null) {189      if (childFlow) closeFlow();190      exitContainers(0);191      effects.consume(code);192      return;193    }194    childFlow = childFlow || self.parser.flow(self.now());195    effects.enter("chunkFlow", {196      _tokenizer: childFlow,197      contentType: "flow",198      previous: childToken199    });200    return flowContinue(code);201  }202 203  /** @type {State} */204  function flowContinue(code) {205    if (code === null) {206      writeToChild(effects.exit("chunkFlow"), true);207      exitContainers(0);208      effects.consume(code);209      return;210    }211    if (markdownLineEnding(code)) {212      effects.consume(code);213      writeToChild(effects.exit("chunkFlow"));214      // Get ready for the next line.215      continued = 0;216      self.interrupt = undefined;217      return start;218    }219    effects.consume(code);220    return flowContinue;221  }222 223  /**224   * @param {Token} token225   *   Token.226   * @param {boolean | undefined} [endOfFile]227   *   Whether the token is at the end of the file (default: `false`).228   * @returns {undefined}229   *   Nothing.230   */231  function writeToChild(token, endOfFile) {232    const stream = self.sliceStream(token);233    if (endOfFile) stream.push(null);234    token.previous = childToken;235    if (childToken) childToken.next = token;236    childToken = token;237    childFlow.defineSkip(token.start);238    childFlow.write(stream);239 240    // Alright, so we just added a lazy line:241    //242    // ```markdown243    // > a244    // b.245    //246    // Or:247    //248    // > ~~~c249    // d250    //251    // Or:252    //253    // > | e |254    // f255    // ```256    //257    // The construct in the second example (fenced code) does not accept lazy258    // lines, so it marked itself as done at the end of its first line, and259    // then the content construct parses `d`.260    // Most constructs in markdown match on the first line: if the first line261    // forms a construct, a non-lazy line can’t “unmake” it.262    //263    // The construct in the third example is potentially a GFM table, and264    // those are *weird*.265    // It *could* be a table, from the first line, if the following line266    // matches a condition.267    // In this case, that second line is lazy, which “unmakes” the first line268    // and turns the whole into one content block.269    //270    // We’ve now parsed the non-lazy and the lazy line, and can figure out271    // whether the lazy line started a new flow block.272    // If it did, we exit the current containers between the two flow blocks.273    if (self.parser.lazy[token.start.line]) {274      let index = childFlow.events.length;275      while (index--) {276        if (277        // The token starts before the line ending…278        childFlow.events[index][1].start.offset < lineStartOffset && (279        // …and either is not ended yet…280        !childFlow.events[index][1].end ||281        // …or ends after it.282        childFlow.events[index][1].end.offset > lineStartOffset)) {283          // Exit: there’s still something open, which means it’s a lazy line284          // part of something.285          return;286        }287      }288 289      // Note: this algorithm for moving events around is similar to the290      // algorithm when closing flow in `documentContinue`.291      const indexBeforeExits = self.events.length;292      let indexBeforeFlow = indexBeforeExits;293      /** @type {boolean | undefined} */294      let seen;295      /** @type {Point | undefined} */296      let point;297 298      // Find the previous chunk (the one before the lazy line).299      while (indexBeforeFlow--) {300        if (self.events[indexBeforeFlow][0] === 'exit' && self.events[indexBeforeFlow][1].type === "chunkFlow") {301          if (seen) {302            point = self.events[indexBeforeFlow][1].end;303            break;304          }305          seen = true;306        }307      }308      exitContainers(continued);309 310      // Fix positions.311      index = indexBeforeExits;312      while (index < self.events.length) {313        self.events[index][1].end = {314          ...point315        };316        index++;317      }318 319      // Inject the exits earlier (they’re still also at the end).320      splice(self.events, indexBeforeFlow + 1, 0, self.events.slice(indexBeforeExits));321 322      // Discard the duplicate exits.323      self.events.length = index;324    }325  }326 327  /**328   * @param {number} size329   *   Size.330   * @returns {undefined}331   *   Nothing.332   */333  function exitContainers(size) {334    let index = stack.length;335 336    // Exit open containers.337    while (index-- > size) {338      const entry = stack[index];339      self.containerState = entry[1];340      entry[0].exit.call(self, effects);341    }342    stack.length = size;343  }344  function closeFlow() {345    childFlow.write([null]);346    childToken = undefined;347    childFlow = undefined;348    self.containerState._closeFlow = undefined;349  }350}351 352/**353 * @this {TokenizeContext}354 *   Context.355 * @type {Tokenizer}356 *   Tokenizer.357 */358function tokenizeContainer(effects, ok, nok) {359  // Always populated by defaults.360 361  return factorySpace(effects, effects.attempt(this.parser.constructs.document, ok, nok), "linePrefix", this.parser.constructs.disable.null.includes('codeIndented') ? undefined : 4);362}