Brunobkr/llama.cpp_AlgMor24_github

1import { DATA_POINTER_POS_32, PROGRAM_OFFSET } from "../generated/constants.js";2import { RawTransferData } from "../generated/lazy/constructors.js";3import { walkProgram } from "../generated/lazy/walk.js";4import { parseAsyncRawImpl, parseSyncRawImpl, returnBufferToCache } from "./common.js";5import { TOKEN } from "./lazy-common.js";6import { getVisitorsArr } from "./visitor.js";7export { Visitor } from "./visitor.js";8 9/**10 * Parse JS/TS source synchronously on current thread.11 *12 * The data in buffer is not deserialized. Is deserialized to JS objects lazily, when accessing the13 * properties of objects.14 *15 * e.g. `program` in returned object is an instance of `Program` class, with getters for `start`, `end`,16 * `body` etc.17 *18 * Returned object contains a `visit` function which can be used to visit the AST with a `Visitor`19 * (`Visitor` class can be obtained by calling `experimentalGetLazyVisitor()`).20 *21 * Returned object contains a `dispose` method. When finished with this AST, it's advisable to call22 * `dispose`, to return the buffer to the cache, so it can be reused.23 * Garbage collector should do this anyway at some point, but on an unpredictable schedule,24 * so it's preferable to call `dispose` manually, to ensure the buffer can be reused immediately.25 *26 * @param {string} filename - Filename27 * @param {string} sourceText - Source text of file28 * @param {Object} options - Parsing options29 * @returns {Object} - Object with property getters for `program`, `module`, `comments`, and `errors`,30 *   and `dispose` and `visit` methods31 */32export function parseSyncLazy(filename, sourceText, options) {33  return parseSyncRawImpl(filename, sourceText, options, construct);34}35 36/**37 * Parse JS/TS source asynchronously on a separate thread.38 *39 * The data in buffer is not deserialized. Is deserialized to JS objects lazily, when accessing the40 * properties of objects.41 *42 * e.g. `program` in returned object is an instance of `Program` class, with getters for `start`, `end`,43 * `body` etc.44 *45 * Because this function does not deserialize the AST, unlike `parse`, very little work happens46 * on current thread in this function. Deserialization work only occurs when properties of the objects47 * are accessed.48 *49 * Returned object contains a `visit` function which can be used to visit the AST with a `Visitor`50 * (`Visitor` class can be obtained by calling `experimentalGetLazyVisitor()`).51 *52 * Returned object contains a `dispose` method. When finished with this AST, it's advisable to call53 * `dispose`, to return the buffer to the cache, so it can be reused.54 * Garbage collector should do this anyway at some point, but on an unpredictable schedule,55 * so it's preferable to call `dispose` manually, to ensure the buffer can be reused immediately.56 *57 * @param {string} filename - Filename58 * @param {string} sourceText - Source text of file59 * @param {Object} options - Parsing options60 * @returns {Object} - Object with property getters for `program`, `module`, `comments`, and `errors`,61 *   and `dispose` and `visit` methods62 */63export function parse(filename, sourceText, options) {64  return parseAsyncRawImpl(filename, sourceText, options, construct);65}66 67// Registry for buffers which are held by lazily-deserialized ASTs.68// Returns buffer to cache when the `ast` wrapper is garbage collected.69//70// Check for existence of `FinalizationRegistry`, to avoid errors on old versions of NodeJS71// which don't support it. e.g. Prettier supports NodeJS v14.72// Raw transfer is disabled on NodeJS before v22, so it doesn't matter if this is `null` on old NodeJS73// - it'll never be accessed in that case.74const bufferRecycleRegistry =75  typeof FinalizationRegistry === "undefined"76    ? null77    : new FinalizationRegistry(returnBufferToCache);78 79/**80 * Get an object with getters which lazy deserialize AST and other data from buffer.81 *82 * Object also includes `dispose` and `visit` functions.83 *84 * @param {Uint8Array} buffer - Buffer containing AST in raw form85 * @param {string} sourceText - Source for the file86 * @param {number} sourceByteLen - Length of source text in UTF-8 bytes87 * @param {Object} _options - Parsing options88 * @returns {Object} - Object with property getters for `program`, `module`, `comments`, and `errors`,89 *   and `dispose` and `visit` methods90 */91function construct(buffer, sourceText, sourceByteLen, _options) {92  // Create AST object93  const sourceIsAscii = sourceText.length === sourceByteLen;94  const ast = { buffer, sourceText, sourceByteLen, sourceIsAscii, nodes: new Map(), token: TOKEN };95 96  // Register `ast` with the recycle registry so buffer is returned to cache97  // when `ast` is garbage collected98  bufferRecycleRegistry.register(ast, buffer, ast);99 100  // Get root data class instance101  const rawDataPos = buffer.int32[DATA_POINTER_POS_32];102  const data = new RawTransferData(rawDataPos, ast);103 104  return {105    get program() {106      return data.program;107    },108    get module() {109      return data.module;110    },111    get comments() {112      return data.comments;113    },114    get errors() {115      return data.errors;116    },117    dispose: dispose.bind(null, ast),118    visit(visitor) {119      walkProgram(rawDataPos + PROGRAM_OFFSET, ast, getVisitorsArr(visitor));120    },121  };122}123 124/**125 * Dispose of this AST.126 *127 * After calling this method, trying to read any nodes from this AST may cause an error.128 *129 * Buffer is returned to the cache to be reused.130 *131 * The buffer would be returned to the cache anyway, once all nodes of the AST are garbage collected,132 * but calling `dispose` is preferable, as it will happen immediately.133 * Otherwise, garbage collector may take time to collect the `ast` object, and new buffers may be created134 * in the meantime, when we could have reused this one.135 *136 * @param {Object} ast - AST object containing buffer etc137 * @returns {undefined}138 */139function dispose(ast) {140  // Return buffer to cache, to be reused141  returnBufferToCache(ast.buffer);142 143  // Remove connection between `ast` and the buffer144  ast.buffer = null;145 146  // Clear other contents of `ast`, so they can be garbage collected147  ast.sourceText = null;148  ast.nodes = null;149 150  // Remove `ast` from recycling register.151  // When `ast` is garbage collected, there's no longer any action to be taken.152  bufferRecycleRegistry.unregister(ast);153}154