Brunobkr/llama.cpp_AlgMor24_github

1"use strict";2 3exports.__esModule = true;4exports["default"] = tokenize;5exports.FIELDS = void 0;6 7var t = _interopRequireWildcard(require("./tokenTypes"));8 9var _unescapable, _wordDelimiters;10 11function _getRequireWildcardCache() { if (typeof WeakMap !== "function") return null; var cache = new WeakMap(); _getRequireWildcardCache = function _getRequireWildcardCache() { return cache; }; return cache; }12 13function _interopRequireWildcard(obj) { if (obj && obj.__esModule) { return obj; } if (obj === null || typeof obj !== "object" && typeof obj !== "function") { return { "default": obj }; } var cache = _getRequireWildcardCache(); if (cache && cache.has(obj)) { return cache.get(obj); } var newObj = {}; var hasPropertyDescriptor = Object.defineProperty && Object.getOwnPropertyDescriptor; for (var key in obj) { if (Object.prototype.hasOwnProperty.call(obj, key)) { var desc = hasPropertyDescriptor ? Object.getOwnPropertyDescriptor(obj, key) : null; if (desc && (desc.get || desc.set)) { Object.defineProperty(newObj, key, desc); } else { newObj[key] = obj[key]; } } } newObj["default"] = obj; if (cache) { cache.set(obj, newObj); } return newObj; }14 15var unescapable = (_unescapable = {}, _unescapable[t.tab] = true, _unescapable[t.newline] = true, _unescapable[t.cr] = true, _unescapable[t.feed] = true, _unescapable);16var wordDelimiters = (_wordDelimiters = {}, _wordDelimiters[t.space] = true, _wordDelimiters[t.tab] = true, _wordDelimiters[t.newline] = true, _wordDelimiters[t.cr] = true, _wordDelimiters[t.feed] = true, _wordDelimiters[t.ampersand] = true, _wordDelimiters[t.asterisk] = true, _wordDelimiters[t.bang] = true, _wordDelimiters[t.comma] = true, _wordDelimiters[t.colon] = true, _wordDelimiters[t.semicolon] = true, _wordDelimiters[t.openParenthesis] = true, _wordDelimiters[t.closeParenthesis] = true, _wordDelimiters[t.openSquare] = true, _wordDelimiters[t.closeSquare] = true, _wordDelimiters[t.singleQuote] = true, _wordDelimiters[t.doubleQuote] = true, _wordDelimiters[t.plus] = true, _wordDelimiters[t.pipe] = true, _wordDelimiters[t.tilde] = true, _wordDelimiters[t.greaterThan] = true, _wordDelimiters[t.equals] = true, _wordDelimiters[t.dollar] = true, _wordDelimiters[t.caret] = true, _wordDelimiters[t.slash] = true, _wordDelimiters);17var hex = {};18var hexChars = "0123456789abcdefABCDEF";19 20for (var i = 0; i < hexChars.length; i++) {21  hex[hexChars.charCodeAt(i)] = true;22}23/**24 *  Returns the last index of the bar css word25 * @param {string} css The string in which the word begins26 * @param {number} start The index into the string where word's first letter occurs27 */28 29 30function consumeWord(css, start) {31  var next = start;32  var code;33 34  do {35    code = css.charCodeAt(next);36 37    if (wordDelimiters[code]) {38      return next - 1;39    } else if (code === t.backslash) {40      next = consumeEscape(css, next) + 1;41    } else {42      // All other characters are part of the word43      next++;44    }45  } while (next < css.length);46 47  return next - 1;48}49/**50 *  Returns the last index of the escape sequence51 * @param {string} css The string in which the sequence begins52 * @param {number} start The index into the string where escape character (`\`) occurs.53 */54 55 56function consumeEscape(css, start) {57  var next = start;58  var code = css.charCodeAt(next + 1);59 60  if (unescapable[code]) {// just consume the escape char61  } else if (hex[code]) {62    var hexDigits = 0; // consume up to 6 hex chars63 64    do {65      next++;66      hexDigits++;67      code = css.charCodeAt(next + 1);68    } while (hex[code] && hexDigits < 6); // if fewer than 6 hex chars, a trailing space ends the escape69 70 71    if (hexDigits < 6 && code === t.space) {72      next++;73    }74  } else {75    // the next char is part of the current word76    next++;77  }78 79  return next;80}81 82var FIELDS = {83  TYPE: 0,84  START_LINE: 1,85  START_COL: 2,86  END_LINE: 3,87  END_COL: 4,88  START_POS: 5,89  END_POS: 690};91exports.FIELDS = FIELDS;92 93function tokenize(input) {94  var tokens = [];95  var css = input.css.valueOf();96  var _css = css,97      length = _css.length;98  var offset = -1;99  var line = 1;100  var start = 0;101  var end = 0;102  var code, content, endColumn, endLine, escaped, escapePos, last, lines, next, nextLine, nextOffset, quote, tokenType;103 104  function unclosed(what, fix) {105    if (input.safe) {106      // fyi: this is never set to true.107      css += fix;108      next = css.length - 1;109    } else {110      throw input.error('Unclosed ' + what, line, start - offset, start);111    }112  }113 114  while (start < length) {115    code = css.charCodeAt(start);116 117    if (code === t.newline) {118      offset = start;119      line += 1;120    }121 122    switch (code) {123      case t.space:124      case t.tab:125      case t.newline:126      case t.cr:127      case t.feed:128        next = start;129 130        do {131          next += 1;132          code = css.charCodeAt(next);133 134          if (code === t.newline) {135            offset = next;136            line += 1;137          }138        } while (code === t.space || code === t.newline || code === t.tab || code === t.cr || code === t.feed);139 140        tokenType = t.space;141        endLine = line;142        endColumn = next - offset - 1;143        end = next;144        break;145 146      case t.plus:147      case t.greaterThan:148      case t.tilde:149      case t.pipe:150        next = start;151 152        do {153          next += 1;154          code = css.charCodeAt(next);155        } while (code === t.plus || code === t.greaterThan || code === t.tilde || code === t.pipe);156 157        tokenType = t.combinator;158        endLine = line;159        endColumn = start - offset;160        end = next;161        break;162      // Consume these characters as single tokens.163 164      case t.asterisk:165      case t.ampersand:166      case t.bang:167      case t.comma:168      case t.equals:169      case t.dollar:170      case t.caret:171      case t.openSquare:172      case t.closeSquare:173      case t.colon:174      case t.semicolon:175      case t.openParenthesis:176      case t.closeParenthesis:177        next = start;178        tokenType = code;179        endLine = line;180        endColumn = start - offset;181        end = next + 1;182        break;183 184      case t.singleQuote:185      case t.doubleQuote:186        quote = code === t.singleQuote ? "'" : '"';187        next = start;188 189        do {190          escaped = false;191          next = css.indexOf(quote, next + 1);192 193          if (next === -1) {194            unclosed('quote', quote);195          }196 197          escapePos = next;198 199          while (css.charCodeAt(escapePos - 1) === t.backslash) {200            escapePos -= 1;201            escaped = !escaped;202          }203        } while (escaped);204 205        tokenType = t.str;206        endLine = line;207        endColumn = start - offset;208        end = next + 1;209        break;210 211      default:212        if (code === t.slash && css.charCodeAt(start + 1) === t.asterisk) {213          next = css.indexOf('*/', start + 2) + 1;214 215          if (next === 0) {216            unclosed('comment', '*/');217          }218 219          content = css.slice(start, next + 1);220          lines = content.split('\n');221          last = lines.length - 1;222 223          if (last > 0) {224            nextLine = line + last;225            nextOffset = next - lines[last].length;226          } else {227            nextLine = line;228            nextOffset = offset;229          }230 231          tokenType = t.comment;232          line = nextLine;233          endLine = nextLine;234          endColumn = next - nextOffset;235        } else if (code === t.slash) {236          next = start;237          tokenType = code;238          endLine = line;239          endColumn = start - offset;240          end = next + 1;241        } else {242          next = consumeWord(css, start);243          tokenType = t.word;244          endLine = line;245          endColumn = next - offset;246        }247 248        end = next + 1;249        break;250    } // Ensure that the token structure remains consistent251 252 253    tokens.push([tokenType, // [0] Token type254    line, // [1] Starting line255    start - offset, // [2] Starting column256    endLine, // [3] Ending line257    endColumn, // [4] Ending column258    start, // [5] Start position / Source index259    end // [6] End position260    ]); // Reset offset for the next token261 262    if (nextOffset) {263      offset = nextOffset;264      nextOffset = null;265    }266 267    start = end;268  }269 270  return tokens;271}