Brunobkr/llama.cpp_AlgMor24_github

1/**2 * @import {Event, Extension, Point, Resolver, State, Token, TokenizeContext, Tokenizer} from 'micromark-util-types'3 */4 5/**6 * @typedef {[number, number, number, number]} Range7 *   Cell info.8 *9 * @typedef {0 | 1 | 2 | 3} RowKind10 *   Where we are: `1` for head row, `2` for delimiter row, `3` for body row.11 */12 13import { factorySpace } from 'micromark-factory-space';14import { markdownLineEnding, markdownLineEndingOrSpace, markdownSpace } from 'micromark-util-character';15import { EditMap } from './edit-map.js';16import { gfmTableAlign } from './infer.js';17 18/**19 * Create an HTML extension for `micromark` to support GitHub tables syntax.20 *21 * @returns {Extension}22 *   Extension for `micromark` that can be passed in `extensions` to enable GFM23 *   table syntax.24 */25export function gfmTable() {26  return {27    flow: {28      null: {29        name: 'table',30        tokenize: tokenizeTable,31        resolveAll: resolveTable32      }33    }34  };35}36 37/**38 * @this {TokenizeContext}39 * @type {Tokenizer}40 */41function tokenizeTable(effects, ok, nok) {42  const self = this;43  let size = 0;44  let sizeB = 0;45  /** @type {boolean | undefined} */46  let seen;47  return start;48 49  /**50   * Start of a GFM table.51   *52   * If there is a valid table row or table head before, then we try to parse53   * another row.54   * Otherwise, we try to parse a head.55   *56   * ```markdown57   * > | | a |58   *     ^59   *   | | - |60   * > | | b |61   *     ^62   * ```63   * @type {State}64   */65  function start(code) {66    let index = self.events.length - 1;67    while (index > -1) {68      const type = self.events[index][1].type;69      if (type === "lineEnding" ||70      // Note: markdown-rs uses `whitespace` instead of `linePrefix`71      type === "linePrefix") index--;else break;72    }73    const tail = index > -1 ? self.events[index][1].type : null;74    const next = tail === 'tableHead' || tail === 'tableRow' ? bodyRowStart : headRowBefore;75 76    // Don’t allow lazy body rows.77    if (next === bodyRowStart && self.parser.lazy[self.now().line]) {78      return nok(code);79    }80    return next(code);81  }82 83  /**84   * Before table head row.85   *86   * ```markdown87   * > | | a |88   *     ^89   *   | | - |90   *   | | b |91   * ```92   *93   * @type {State}94   */95  function headRowBefore(code) {96    effects.enter('tableHead');97    effects.enter('tableRow');98    return headRowStart(code);99  }100 101  /**102   * Before table head row, after whitespace.103   *104   * ```markdown105   * > | | a |106   *     ^107   *   | | - |108   *   | | b |109   * ```110   *111   * @type {State}112   */113  function headRowStart(code) {114    if (code === 124) {115      return headRowBreak(code);116    }117 118    // To do: micromark-js should let us parse our own whitespace in extensions,119    // like `markdown-rs`:120    //121    // ```js122    // // 4+ spaces.123    // if (markdownSpace(code)) {124    //   return nok(code)125    // }126    // ```127 128    seen = true;129    // Count the first character, that isn’t a pipe, double.130    sizeB += 1;131    return headRowBreak(code);132  }133 134  /**135   * At break in table head row.136   *137   * ```markdown138   * > | | a |139   *     ^140   *       ^141   *         ^142   *   | | - |143   *   | | b |144   * ```145   *146   * @type {State}147   */148  function headRowBreak(code) {149    if (code === null) {150      // Note: in `markdown-rs`, we need to reset, in `micromark-js` we don‘t.151      return nok(code);152    }153    if (markdownLineEnding(code)) {154      // If anything other than one pipe (ignoring whitespace) was used, it’s fine.155      if (sizeB > 1) {156        sizeB = 0;157        // To do: check if this works.158        // Feel free to interrupt:159        self.interrupt = true;160        effects.exit('tableRow');161        effects.enter("lineEnding");162        effects.consume(code);163        effects.exit("lineEnding");164        return headDelimiterStart;165      }166 167      // Note: in `markdown-rs`, we need to reset, in `micromark-js` we don‘t.168      return nok(code);169    }170    if (markdownSpace(code)) {171      // To do: check if this is fine.172      // effects.attempt(State::Next(StateName::GfmTableHeadRowBreak), State::Nok)173      // State::Retry(space_or_tab(tokenizer))174      return factorySpace(effects, headRowBreak, "whitespace")(code);175    }176    sizeB += 1;177    if (seen) {178      seen = false;179      // Header cell count.180      size += 1;181    }182    if (code === 124) {183      effects.enter('tableCellDivider');184      effects.consume(code);185      effects.exit('tableCellDivider');186      // Whether a delimiter was seen.187      seen = true;188      return headRowBreak;189    }190 191    // Anything else is cell data.192    effects.enter("data");193    return headRowData(code);194  }195 196  /**197   * In table head row data.198   *199   * ```markdown200   * > | | a |201   *       ^202   *   | | - |203   *   | | b |204   * ```205   *206   * @type {State}207   */208  function headRowData(code) {209    if (code === null || code === 124 || markdownLineEndingOrSpace(code)) {210      effects.exit("data");211      return headRowBreak(code);212    }213    effects.consume(code);214    return code === 92 ? headRowEscape : headRowData;215  }216 217  /**218   * In table head row escape.219   *220   * ```markdown221   * > | | a\-b |222   *         ^223   *   | | ---- |224   *   | | c    |225   * ```226   *227   * @type {State}228   */229  function headRowEscape(code) {230    if (code === 92 || code === 124) {231      effects.consume(code);232      return headRowData;233    }234    return headRowData(code);235  }236 237  /**238   * Before delimiter row.239   *240   * ```markdown241   *   | | a |242   * > | | - |243   *     ^244   *   | | b |245   * ```246   *247   * @type {State}248   */249  function headDelimiterStart(code) {250    // Reset `interrupt`.251    self.interrupt = false;252 253    // Note: in `markdown-rs`, we need to handle piercing here too.254    if (self.parser.lazy[self.now().line]) {255      return nok(code);256    }257    effects.enter('tableDelimiterRow');258    // Track if we’ve seen a `:` or `|`.259    seen = false;260    if (markdownSpace(code)) {261      return factorySpace(effects, headDelimiterBefore, "linePrefix", self.parser.constructs.disable.null.includes('codeIndented') ? undefined : 4)(code);262    }263    return headDelimiterBefore(code);264  }265 266  /**267   * Before delimiter row, after optional whitespace.268   *269   * Reused when a `|` is found later, to parse another cell.270   *271   * ```markdown272   *   | | a |273   * > | | - |274   *     ^275   *   | | b |276   * ```277   *278   * @type {State}279   */280  function headDelimiterBefore(code) {281    if (code === 45 || code === 58) {282      return headDelimiterValueBefore(code);283    }284    if (code === 124) {285      seen = true;286      // If we start with a pipe, we open a cell marker.287      effects.enter('tableCellDivider');288      effects.consume(code);289      effects.exit('tableCellDivider');290      return headDelimiterCellBefore;291    }292 293    // More whitespace / empty row not allowed at start.294    return headDelimiterNok(code);295  }296 297  /**298   * After `|`, before delimiter cell.299   *300   * ```markdown301   *   | | a |302   * > | | - |303   *      ^304   * ```305   *306   * @type {State}307   */308  function headDelimiterCellBefore(code) {309    if (markdownSpace(code)) {310      return factorySpace(effects, headDelimiterValueBefore, "whitespace")(code);311    }312    return headDelimiterValueBefore(code);313  }314 315  /**316   * Before delimiter cell value.317   *318   * ```markdown319   *   | | a |320   * > | | - |321   *       ^322   * ```323   *324   * @type {State}325   */326  function headDelimiterValueBefore(code) {327    // Align: left.328    if (code === 58) {329      sizeB += 1;330      seen = true;331      effects.enter('tableDelimiterMarker');332      effects.consume(code);333      effects.exit('tableDelimiterMarker');334      return headDelimiterLeftAlignmentAfter;335    }336 337    // Align: none.338    if (code === 45) {339      sizeB += 1;340      // To do: seems weird that this *isn’t* left aligned, but that state is used?341      return headDelimiterLeftAlignmentAfter(code);342    }343    if (code === null || markdownLineEnding(code)) {344      return headDelimiterCellAfter(code);345    }346    return headDelimiterNok(code);347  }348 349  /**350   * After delimiter cell left alignment marker.351   *352   * ```markdown353   *   | | a  |354   * > | | :- |355   *        ^356   * ```357   *358   * @type {State}359   */360  function headDelimiterLeftAlignmentAfter(code) {361    if (code === 45) {362      effects.enter('tableDelimiterFiller');363      return headDelimiterFiller(code);364    }365 366    // Anything else is not ok after the left-align colon.367    return headDelimiterNok(code);368  }369 370  /**371   * In delimiter cell filler.372   *373   * ```markdown374   *   | | a |375   * > | | - |376   *       ^377   * ```378   *379   * @type {State}380   */381  function headDelimiterFiller(code) {382    if (code === 45) {383      effects.consume(code);384      return headDelimiterFiller;385    }386 387    // Align is `center` if it was `left`, `right` otherwise.388    if (code === 58) {389      seen = true;390      effects.exit('tableDelimiterFiller');391      effects.enter('tableDelimiterMarker');392      effects.consume(code);393      effects.exit('tableDelimiterMarker');394      return headDelimiterRightAlignmentAfter;395    }396    effects.exit('tableDelimiterFiller');397    return headDelimiterRightAlignmentAfter(code);398  }399 400  /**401   * After delimiter cell right alignment marker.402   *403   * ```markdown404   *   | |  a |405   * > | | -: |406   *         ^407   * ```408   *409   * @type {State}410   */411  function headDelimiterRightAlignmentAfter(code) {412    if (markdownSpace(code)) {413      return factorySpace(effects, headDelimiterCellAfter, "whitespace")(code);414    }415    return headDelimiterCellAfter(code);416  }417 418  /**419   * After delimiter cell.420   *421   * ```markdown422   *   | |  a |423   * > | | -: |424   *          ^425   * ```426   *427   * @type {State}428   */429  function headDelimiterCellAfter(code) {430    if (code === 124) {431      return headDelimiterBefore(code);432    }433    if (code === null || markdownLineEnding(code)) {434      // Exit when:435      // * there was no `:` or `|` at all (it’s a thematic break or setext436      //   underline instead)437      // * the header cell count is not the delimiter cell count438      if (!seen || size !== sizeB) {439        return headDelimiterNok(code);440      }441 442      // Note: in markdown-rs`, a reset is needed here.443      effects.exit('tableDelimiterRow');444      effects.exit('tableHead');445      // To do: in `markdown-rs`, resolvers need to be registered manually.446      // effects.register_resolver(ResolveName::GfmTable)447      return ok(code);448    }449    return headDelimiterNok(code);450  }451 452  /**453   * In delimiter row, at a disallowed byte.454   *455   * ```markdown456   *   | | a |457   * > | | x |458   *       ^459   * ```460   *461   * @type {State}462   */463  function headDelimiterNok(code) {464    // Note: in `markdown-rs`, we need to reset, in `micromark-js` we don‘t.465    return nok(code);466  }467 468  /**469   * Before table body row.470   *471   * ```markdown472   *   | | a |473   *   | | - |474   * > | | b |475   *     ^476   * ```477   *478   * @type {State}479   */480  function bodyRowStart(code) {481    // Note: in `markdown-rs` we need to manually take care of a prefix,482    // but in `micromark-js` that is done for us, so if we’re here, we’re483    // never at whitespace.484    effects.enter('tableRow');485    return bodyRowBreak(code);486  }487 488  /**489   * At break in table body row.490   *491   * ```markdown492   *   | | a |493   *   | | - |494   * > | | b |495   *     ^496   *       ^497   *         ^498   * ```499   *500   * @type {State}501   */502  function bodyRowBreak(code) {503    if (code === 124) {504      effects.enter('tableCellDivider');505      effects.consume(code);506      effects.exit('tableCellDivider');507      return bodyRowBreak;508    }509    if (code === null || markdownLineEnding(code)) {510      effects.exit('tableRow');511      return ok(code);512    }513    if (markdownSpace(code)) {514      return factorySpace(effects, bodyRowBreak, "whitespace")(code);515    }516 517    // Anything else is cell content.518    effects.enter("data");519    return bodyRowData(code);520  }521 522  /**523   * In table body row data.524   *525   * ```markdown526   *   | | a |527   *   | | - |528   * > | | b |529   *       ^530   * ```531   *532   * @type {State}533   */534  function bodyRowData(code) {535    if (code === null || code === 124 || markdownLineEndingOrSpace(code)) {536      effects.exit("data");537      return bodyRowBreak(code);538    }539    effects.consume(code);540    return code === 92 ? bodyRowEscape : bodyRowData;541  }542 543  /**544   * In table body row escape.545   *546   * ```markdown547   *   | | a    |548   *   | | ---- |549   * > | | b\-c |550   *         ^551   * ```552   *553   * @type {State}554   */555  function bodyRowEscape(code) {556    if (code === 92 || code === 124) {557      effects.consume(code);558      return bodyRowData;559    }560    return bodyRowData(code);561  }562}563 564/** @type {Resolver} */565 566function resolveTable(events, context) {567  let index = -1;568  let inFirstCellAwaitingPipe = true;569  /** @type {RowKind} */570  let rowKind = 0;571  /** @type {Range} */572  let lastCell = [0, 0, 0, 0];573  /** @type {Range} */574  let cell = [0, 0, 0, 0];575  let afterHeadAwaitingFirstBodyRow = false;576  let lastTableEnd = 0;577  /** @type {Token | undefined} */578  let currentTable;579  /** @type {Token | undefined} */580  let currentBody;581  /** @type {Token | undefined} */582  let currentCell;583  const map = new EditMap();584  while (++index < events.length) {585    const event = events[index];586    const token = event[1];587    if (event[0] === 'enter') {588      // Start of head.589      if (token.type === 'tableHead') {590        afterHeadAwaitingFirstBodyRow = false;591 592        // Inject previous (body end and) table end.593        if (lastTableEnd !== 0) {594          flushTableEnd(map, context, lastTableEnd, currentTable, currentBody);595          currentBody = undefined;596          lastTableEnd = 0;597        }598 599        // Inject table start.600        currentTable = {601          type: 'table',602          start: Object.assign({}, token.start),603          // Note: correct end is set later.604          end: Object.assign({}, token.end)605        };606        map.add(index, 0, [['enter', currentTable, context]]);607      } else if (token.type === 'tableRow' || token.type === 'tableDelimiterRow') {608        inFirstCellAwaitingPipe = true;609        currentCell = undefined;610        lastCell = [0, 0, 0, 0];611        cell = [0, index + 1, 0, 0];612 613        // Inject table body start.614        if (afterHeadAwaitingFirstBodyRow) {615          afterHeadAwaitingFirstBodyRow = false;616          currentBody = {617            type: 'tableBody',618            start: Object.assign({}, token.start),619            // Note: correct end is set later.620            end: Object.assign({}, token.end)621          };622          map.add(index, 0, [['enter', currentBody, context]]);623        }624        rowKind = token.type === 'tableDelimiterRow' ? 2 : currentBody ? 3 : 1;625      }626      // Cell data.627      else if (rowKind && (token.type === "data" || token.type === 'tableDelimiterMarker' || token.type === 'tableDelimiterFiller')) {628        inFirstCellAwaitingPipe = false;629 630        // First value in cell.631        if (cell[2] === 0) {632          if (lastCell[1] !== 0) {633            cell[0] = cell[1];634            currentCell = flushCell(map, context, lastCell, rowKind, undefined, currentCell);635            lastCell = [0, 0, 0, 0];636          }637          cell[2] = index;638        }639      } else if (token.type === 'tableCellDivider') {640        if (inFirstCellAwaitingPipe) {641          inFirstCellAwaitingPipe = false;642        } else {643          if (lastCell[1] !== 0) {644            cell[0] = cell[1];645            currentCell = flushCell(map, context, lastCell, rowKind, undefined, currentCell);646          }647          lastCell = cell;648          cell = [lastCell[1], index, 0, 0];649        }650      }651    }652    // Exit events.653    else if (token.type === 'tableHead') {654      afterHeadAwaitingFirstBodyRow = true;655      lastTableEnd = index;656    } else if (token.type === 'tableRow' || token.type === 'tableDelimiterRow') {657      lastTableEnd = index;658      if (lastCell[1] !== 0) {659        cell[0] = cell[1];660        currentCell = flushCell(map, context, lastCell, rowKind, index, currentCell);661      } else if (cell[1] !== 0) {662        currentCell = flushCell(map, context, cell, rowKind, index, currentCell);663      }664      rowKind = 0;665    } else if (rowKind && (token.type === "data" || token.type === 'tableDelimiterMarker' || token.type === 'tableDelimiterFiller')) {666      cell[3] = index;667    }668  }669  if (lastTableEnd !== 0) {670    flushTableEnd(map, context, lastTableEnd, currentTable, currentBody);671  }672  map.consume(context.events);673 674  // To do: move this into `html`, when events are exposed there.675  // That’s what `markdown-rs` does.676  // That needs updates to `mdast-util-gfm-table`.677  index = -1;678  while (++index < context.events.length) {679    const event = context.events[index];680    if (event[0] === 'enter' && event[1].type === 'table') {681      event[1]._align = gfmTableAlign(context.events, index);682    }683  }684  return events;685}686 687/**688 * Generate a cell.689 *690 * @param {EditMap} map691 * @param {Readonly<TokenizeContext>} context692 * @param {Readonly<Range>} range693 * @param {RowKind} rowKind694 * @param {number | undefined} rowEnd695 * @param {Token | undefined} previousCell696 * @returns {Token | undefined}697 */698// eslint-disable-next-line max-params699function flushCell(map, context, range, rowKind, rowEnd, previousCell) {700  // `markdown-rs` uses:701  // rowKind === 2 ? 'tableDelimiterCell' : 'tableCell'702  const groupName = rowKind === 1 ? 'tableHeader' : rowKind === 2 ? 'tableDelimiter' : 'tableData';703  // `markdown-rs` uses:704  // rowKind === 2 ? 'tableDelimiterCellValue' : 'tableCellText'705  const valueName = 'tableContent';706 707  // Insert an exit for the previous cell, if there is one.708  //709  // ```markdown710  // > | | aa | bb | cc |711  //          ^-- exit712  //           ^^^^-- this cell713  // ```714  if (range[0] !== 0) {715    previousCell.end = Object.assign({}, getPoint(context.events, range[0]));716    map.add(range[0], 0, [['exit', previousCell, context]]);717  }718 719  // Insert enter of this cell.720  //721  // ```markdown722  // > | | aa | bb | cc |723  //           ^-- enter724  //           ^^^^-- this cell725  // ```726  const now = getPoint(context.events, range[1]);727  previousCell = {728    type: groupName,729    start: Object.assign({}, now),730    // Note: correct end is set later.731    end: Object.assign({}, now)732  };733  map.add(range[1], 0, [['enter', previousCell, context]]);734 735  // Insert text start at first data start and end at last data end, and736  // remove events between.737  //738  // ```markdown739  // > | | aa | bb | cc |740  //            ^-- enter741  //             ^-- exit742  //           ^^^^-- this cell743  // ```744  if (range[2] !== 0) {745    const relatedStart = getPoint(context.events, range[2]);746    const relatedEnd = getPoint(context.events, range[3]);747    /** @type {Token} */748    const valueToken = {749      type: valueName,750      start: Object.assign({}, relatedStart),751      end: Object.assign({}, relatedEnd)752    };753    map.add(range[2], 0, [['enter', valueToken, context]]);754    if (rowKind !== 2) {755      // Fix positional info on remaining events756      const start = context.events[range[2]];757      const end = context.events[range[3]];758      start[1].end = Object.assign({}, end[1].end);759      start[1].type = "chunkText";760      start[1].contentType = "text";761 762      // Remove if needed.763      if (range[3] > range[2] + 1) {764        const a = range[2] + 1;765        const b = range[3] - range[2] - 1;766        map.add(a, b, []);767      }768    }769    map.add(range[3] + 1, 0, [['exit', valueToken, context]]);770  }771 772  // Insert an exit for the last cell, if at the row end.773  //774  // ```markdown775  // > | | aa | bb | cc |776  //                    ^-- exit777  //               ^^^^^^-- this cell (the last one contains two “between” parts)778  // ```779  if (rowEnd !== undefined) {780    previousCell.end = Object.assign({}, getPoint(context.events, rowEnd));781    map.add(rowEnd, 0, [['exit', previousCell, context]]);782    previousCell = undefined;783  }784  return previousCell;785}786 787/**788 * Generate table end (and table body end).789 *790 * @param {Readonly<EditMap>} map791 * @param {Readonly<TokenizeContext>} context792 * @param {number} index793 * @param {Token} table794 * @param {Token | undefined} tableBody795 */796// eslint-disable-next-line max-params797function flushTableEnd(map, context, index, table, tableBody) {798  /** @type {Array<Event>} */799  const exits = [];800  const related = getPoint(context.events, index);801  if (tableBody) {802    tableBody.end = Object.assign({}, related);803    exits.push(['exit', tableBody, context]);804  }805  table.end = Object.assign({}, related);806  exits.push(['exit', table, context]);807  map.add(index + 1, 0, exits);808}809 810/**811 * @param {Readonly<Array<Event>>} events812 * @param {number} index813 * @returns {Readonly<Point>}814 */815function getPoint(events, index) {816  const event = events[index];817  const side = event[0] === 'enter' ? 'start' : 'end';818  return event[1][side];819}