Brunobkr/llama.cpp_AlgMor24_github

1import { watch as fs_watch, unwatchFile, watchFile } from 'node:fs';2import { realpath as fsrealpath, lstat, open, stat } from 'node:fs/promises';3import { type as osType } from 'node:os';4import * as sp from 'node:path';5export const STR_DATA = 'data';6export const STR_END = 'end';7export const STR_CLOSE = 'close';8export const EMPTY_FN = () => { };9export const IDENTITY_FN = (val) => val;10const pl = process.platform;11export const isWindows = pl === 'win32';12export const isMacos = pl === 'darwin';13export const isLinux = pl === 'linux';14export const isFreeBSD = pl === 'freebsd';15export const isIBMi = osType() === 'OS400';16export const EVENTS = {17    ALL: 'all',18    READY: 'ready',19    ADD: 'add',20    CHANGE: 'change',21    ADD_DIR: 'addDir',22    UNLINK: 'unlink',23    UNLINK_DIR: 'unlinkDir',24    RAW: 'raw',25    ERROR: 'error',26};27const EV = EVENTS;28const THROTTLE_MODE_WATCH = 'watch';29const statMethods = { lstat, stat };30const KEY_LISTENERS = 'listeners';31const KEY_ERR = 'errHandlers';32const KEY_RAW = 'rawEmitters';33const HANDLER_KEYS = [KEY_LISTENERS, KEY_ERR, KEY_RAW];34// prettier-ignore35const binaryExtensions = new Set([36    '3dm', '3ds', '3g2', '3gp', '7z', 'a', 'aac', 'adp', 'afdesign', 'afphoto', 'afpub', 'ai',37    'aif', 'aiff', 'alz', 'ape', 'apk', 'appimage', 'ar', 'arj', 'asf', 'au', 'avi',38    'bak', 'baml', 'bh', 'bin', 'bk', 'bmp', 'btif', 'bz2', 'bzip2',39    'cab', 'caf', 'cgm', 'class', 'cmx', 'cpio', 'cr2', 'cur', 'dat', 'dcm', 'deb', 'dex', 'djvu',40    'dll', 'dmg', 'dng', 'doc', 'docm', 'docx', 'dot', 'dotm', 'dra', 'DS_Store', 'dsk', 'dts',41    'dtshd', 'dvb', 'dwg', 'dxf',42    'ecelp4800', 'ecelp7470', 'ecelp9600', 'egg', 'eol', 'eot', 'epub', 'exe',43    'f4v', 'fbs', 'fh', 'fla', 'flac', 'flatpak', 'fli', 'flv', 'fpx', 'fst', 'fvt',44    'g3', 'gh', 'gif', 'graffle', 'gz', 'gzip',45    'h261', 'h263', 'h264', 'icns', 'ico', 'ief', 'img', 'ipa', 'iso',46    'jar', 'jpeg', 'jpg', 'jpgv', 'jpm', 'jxr', 'key', 'ktx',47    'lha', 'lib', 'lvp', 'lz', 'lzh', 'lzma', 'lzo',48    'm3u', 'm4a', 'm4v', 'mar', 'mdi', 'mht', 'mid', 'midi', 'mj2', 'mka', 'mkv', 'mmr', 'mng',49    'mobi', 'mov', 'movie', 'mp3',50    'mp4', 'mp4a', 'mpeg', 'mpg', 'mpga', 'mxu',51    'nef', 'npx', 'numbers', 'nupkg',52    'o', 'odp', 'ods', 'odt', 'oga', 'ogg', 'ogv', 'otf', 'ott',53    'pages', 'pbm', 'pcx', 'pdb', 'pdf', 'pea', 'pgm', 'pic', 'png', 'pnm', 'pot', 'potm',54    'potx', 'ppa', 'ppam',55    'ppm', 'pps', 'ppsm', 'ppsx', 'ppt', 'pptm', 'pptx', 'psd', 'pya', 'pyc', 'pyo', 'pyv',56    'qt',57    'rar', 'ras', 'raw', 'resources', 'rgb', 'rip', 'rlc', 'rmf', 'rmvb', 'rpm', 'rtf', 'rz',58    's3m', 's7z', 'scpt', 'sgi', 'shar', 'snap', 'sil', 'sketch', 'slk', 'smv', 'snk', 'so',59    'stl', 'suo', 'sub', 'swf',60    'tar', 'tbz', 'tbz2', 'tga', 'tgz', 'thmx', 'tif', 'tiff', 'tlz', 'ttc', 'ttf', 'txz',61    'udf', 'uvh', 'uvi', 'uvm', 'uvp', 'uvs', 'uvu',62    'viv', 'vob',63    'war', 'wav', 'wax', 'wbmp', 'wdp', 'weba', 'webm', 'webp', 'whl', 'wim', 'wm', 'wma',64    'wmv', 'wmx', 'woff', 'woff2', 'wrm', 'wvx',65    'xbm', 'xif', 'xla', 'xlam', 'xls', 'xlsb', 'xlsm', 'xlsx', 'xlt', 'xltm', 'xltx', 'xm',66    'xmind', 'xpi', 'xpm', 'xwd', 'xz',67    'z', 'zip', 'zipx',68]);69const isBinaryPath = (filePath) => binaryExtensions.has(sp.extname(filePath).slice(1).toLowerCase());70// TODO: emit errors properly. Example: EMFILE on Macos.71const foreach = (val, fn) => {72    if (val instanceof Set) {73        val.forEach(fn);74    }75    else {76        fn(val);77    }78};79const addAndConvert = (main, prop, item) => {80    let container = main[prop];81    if (!(container instanceof Set)) {82        main[prop] = container = new Set([container]);83    }84    container.add(item);85};86const clearItem = (cont) => (key) => {87    const set = cont[key];88    if (set instanceof Set) {89        set.clear();90    }91    else {92        delete cont[key];93    }94};95const delFromSet = (main, prop, item) => {96    const container = main[prop];97    if (container instanceof Set) {98        container.delete(item);99    }100    else if (container === item) {101        delete main[prop];102    }103};104const isEmptySet = (val) => (val instanceof Set ? val.size === 0 : !val);105const FsWatchInstances = new Map();106/**107 * Instantiates the fs_watch interface108 * @param path to be watched109 * @param options to be passed to fs_watch110 * @param listener main event handler111 * @param errHandler emits info about errors112 * @param emitRaw emits raw event data113 * @returns {NativeFsWatcher}114 */115function createFsWatchInstance(path, options, listener, errHandler, emitRaw) {116    const handleEvent = (rawEvent, evPath) => {117        listener(path);118        emitRaw(rawEvent, evPath, { watchedPath: path });119        // emit based on events occurring for files from a directory's watcher in120        // case the file's watcher misses it (and rely on throttling to de-dupe)121        if (evPath && path !== evPath) {122            fsWatchBroadcast(sp.resolve(path, evPath), KEY_LISTENERS, sp.join(path, evPath));123        }124    };125    try {126        return fs_watch(path, {127            persistent: options.persistent,128        }, handleEvent);129    }130    catch (error) {131        errHandler(error);132        return undefined;133    }134}135/**136 * Helper for passing fs_watch event data to a collection of listeners137 * @param fullPath absolute path bound to fs_watch instance138 */139const fsWatchBroadcast = (fullPath, listenerType, val1, val2, val3) => {140    const cont = FsWatchInstances.get(fullPath);141    if (!cont)142        return;143    foreach(cont[listenerType], (listener) => {144        listener(val1, val2, val3);145    });146};147/**148 * Instantiates the fs_watch interface or binds listeners149 * to an existing one covering the same file system entry150 * @param path151 * @param fullPath absolute path152 * @param options to be passed to fs_watch153 * @param handlers container for event listener functions154 */155const setFsWatchListener = (path, fullPath, options, handlers) => {156    const { listener, errHandler, rawEmitter } = handlers;157    let cont = FsWatchInstances.get(fullPath);158    let watcher;159    if (!options.persistent) {160        watcher = createFsWatchInstance(path, options, listener, errHandler, rawEmitter);161        if (!watcher)162            return;163        return watcher.close.bind(watcher);164    }165    if (cont) {166        addAndConvert(cont, KEY_LISTENERS, listener);167        addAndConvert(cont, KEY_ERR, errHandler);168        addAndConvert(cont, KEY_RAW, rawEmitter);169    }170    else {171        watcher = createFsWatchInstance(path, options, fsWatchBroadcast.bind(null, fullPath, KEY_LISTENERS), errHandler, // no need to use broadcast here172        fsWatchBroadcast.bind(null, fullPath, KEY_RAW));173        if (!watcher)174            return;175        watcher.on(EV.ERROR, async (error) => {176            const broadcastErr = fsWatchBroadcast.bind(null, fullPath, KEY_ERR);177            if (cont)178                cont.watcherUnusable = true; // documented since Node 10.4.1179            // Workaround for https://github.com/joyent/node/issues/4337180            if (isWindows && error.code === 'EPERM') {181                try {182                    const fd = await open(path, 'r');183                    await fd.close();184                    broadcastErr(error);185                }186                catch (err) {187                    // do nothing188                }189            }190            else {191                broadcastErr(error);192            }193        });194        cont = {195            listeners: listener,196            errHandlers: errHandler,197            rawEmitters: rawEmitter,198            watcher,199        };200        FsWatchInstances.set(fullPath, cont);201    }202    // const index = cont.listeners.indexOf(listener);203    // removes this instance's listeners and closes the underlying fs_watch204    // instance if there are no more listeners left205    return () => {206        delFromSet(cont, KEY_LISTENERS, listener);207        delFromSet(cont, KEY_ERR, errHandler);208        delFromSet(cont, KEY_RAW, rawEmitter);209        if (isEmptySet(cont.listeners)) {210            // Check to protect against issue gh-730.211            // if (cont.watcherUnusable) {212            cont.watcher.close();213            // }214            FsWatchInstances.delete(fullPath);215            HANDLER_KEYS.forEach(clearItem(cont));216            // @ts-ignore217            cont.watcher = undefined;218            Object.freeze(cont);219        }220    };221};222// fs_watchFile helpers223// object to hold per-process fs_watchFile instances224// (may be shared across chokidar FSWatcher instances)225const FsWatchFileInstances = new Map();226/**227 * Instantiates the fs_watchFile interface or binds listeners228 * to an existing one covering the same file system entry229 * @param path to be watched230 * @param fullPath absolute path231 * @param options options to be passed to fs_watchFile232 * @param handlers container for event listener functions233 * @returns closer234 */235const setFsWatchFileListener = (path, fullPath, options, handlers) => {236    const { listener, rawEmitter } = handlers;237    let cont = FsWatchFileInstances.get(fullPath);238    // let listeners = new Set();239    // let rawEmitters = new Set();240    const copts = cont && cont.options;241    if (copts && (copts.persistent < options.persistent || copts.interval > options.interval)) {242        // "Upgrade" the watcher to persistence or a quicker interval.243        // This creates some unlikely edge case issues if the user mixes244        // settings in a very weird way, but solving for those cases245        // doesn't seem worthwhile for the added complexity.246        // listeners = cont.listeners;247        // rawEmitters = cont.rawEmitters;248        unwatchFile(fullPath);249        cont = undefined;250    }251    if (cont) {252        addAndConvert(cont, KEY_LISTENERS, listener);253        addAndConvert(cont, KEY_RAW, rawEmitter);254    }255    else {256        // TODO257        // listeners.add(listener);258        // rawEmitters.add(rawEmitter);259        cont = {260            listeners: listener,261            rawEmitters: rawEmitter,262            options,263            watcher: watchFile(fullPath, options, (curr, prev) => {264                foreach(cont.rawEmitters, (rawEmitter) => {265                    rawEmitter(EV.CHANGE, fullPath, { curr, prev });266                });267                const currmtime = curr.mtimeMs;268                if (curr.size !== prev.size || currmtime > prev.mtimeMs || currmtime === 0) {269                    foreach(cont.listeners, (listener) => listener(path, curr));270                }271            }),272        };273        FsWatchFileInstances.set(fullPath, cont);274    }275    // const index = cont.listeners.indexOf(listener);276    // Removes this instance's listeners and closes the underlying fs_watchFile277    // instance if there are no more listeners left.278    return () => {279        delFromSet(cont, KEY_LISTENERS, listener);280        delFromSet(cont, KEY_RAW, rawEmitter);281        if (isEmptySet(cont.listeners)) {282            FsWatchFileInstances.delete(fullPath);283            unwatchFile(fullPath);284            cont.options = cont.watcher = undefined;285            Object.freeze(cont);286        }287    };288};289/**290 * @mixin291 */292export class NodeFsHandler {293    fsw;294    _boundHandleError;295    constructor(fsW) {296        this.fsw = fsW;297        this._boundHandleError = (error) => fsW._handleError(error);298    }299    /**300     * Watch file for changes with fs_watchFile or fs_watch.301     * @param path to file or dir302     * @param listener on fs change303     * @returns closer for the watcher instance304     */305    _watchWithNodeFs(path, listener) {306        const opts = this.fsw.options;307        const directory = sp.dirname(path);308        const basename = sp.basename(path);309        const parent = this.fsw._getWatchedDir(directory);310        parent.add(basename);311        const absolutePath = sp.resolve(path);312        const options = {313            persistent: opts.persistent,314        };315        if (!listener)316            listener = EMPTY_FN;317        let closer;318        if (opts.usePolling) {319            const enableBin = opts.interval !== opts.binaryInterval;320            options.interval = enableBin && isBinaryPath(basename) ? opts.binaryInterval : opts.interval;321            closer = setFsWatchFileListener(path, absolutePath, options, {322                listener,323                rawEmitter: this.fsw._emitRaw,324            });325        }326        else {327            closer = setFsWatchListener(path, absolutePath, options, {328                listener,329                errHandler: this._boundHandleError,330                rawEmitter: this.fsw._emitRaw,331            });332        }333        return closer;334    }335    /**336     * Watch a file and emit add event if warranted.337     * @returns closer for the watcher instance338     */339    _handleFile(file, stats, initialAdd) {340        if (this.fsw.closed) {341            return;342        }343        const dirname = sp.dirname(file);344        const basename = sp.basename(file);345        const parent = this.fsw._getWatchedDir(dirname);346        // stats is always present347        let prevStats = stats;348        // if the file is already being watched, do nothing349        if (parent.has(basename))350            return;351        const listener = async (path, newStats) => {352            if (!this.fsw._throttle(THROTTLE_MODE_WATCH, file, 5))353                return;354            if (!newStats || newStats.mtimeMs === 0) {355                try {356                    const newStats = await stat(file);357                    if (this.fsw.closed)358                        return;359                    // Check that change event was not fired because of changed only accessTime.360                    const at = newStats.atimeMs;361                    const mt = newStats.mtimeMs;362                    if (!at || at <= mt || mt !== prevStats.mtimeMs) {363                        this.fsw._emit(EV.CHANGE, file, newStats);364                    }365                    if ((isMacos || isLinux || isFreeBSD) && prevStats.ino !== newStats.ino) {366                        this.fsw._closeFile(path);367                        prevStats = newStats;368                        const closer = this._watchWithNodeFs(file, listener);369                        if (closer)370                            this.fsw._addPathCloser(path, closer);371                    }372                    else {373                        prevStats = newStats;374                    }375                }376                catch (error) {377                    // Fix issues where mtime is null but file is still present378                    this.fsw._remove(dirname, basename);379                }380                // add is about to be emitted if file not already tracked in parent381            }382            else if (parent.has(basename)) {383                // Check that change event was not fired because of changed only accessTime.384                const at = newStats.atimeMs;385                const mt = newStats.mtimeMs;386                if (!at || at <= mt || mt !== prevStats.mtimeMs) {387                    this.fsw._emit(EV.CHANGE, file, newStats);388                }389                prevStats = newStats;390            }391        };392        // kick off the watcher393        const closer = this._watchWithNodeFs(file, listener);394        // emit an add event if we're supposed to395        if (!(initialAdd && this.fsw.options.ignoreInitial) && this.fsw._isntIgnored(file)) {396            if (!this.fsw._throttle(EV.ADD, file, 0))397                return;398            this.fsw._emit(EV.ADD, file, stats);399        }400        return closer;401    }402    /**403     * Handle symlinks encountered while reading a dir.404     * @param entry returned by readdirp405     * @param directory path of dir being read406     * @param path of this item407     * @param item basename of this item408     * @returns true if no more processing is needed for this entry.409     */410    async _handleSymlink(entry, directory, path, item) {411        if (this.fsw.closed) {412            return;413        }414        const full = entry.fullPath;415        const dir = this.fsw._getWatchedDir(directory);416        if (!this.fsw.options.followSymlinks) {417            // watch symlink directly (don't follow) and detect changes418            this.fsw._incrReadyCount();419            let linkPath;420            try {421                linkPath = await fsrealpath(path);422            }423            catch (e) {424                this.fsw._emitReady();425                return true;426            }427            if (this.fsw.closed)428                return;429            if (dir.has(item)) {430                if (this.fsw._symlinkPaths.get(full) !== linkPath) {431                    this.fsw._symlinkPaths.set(full, linkPath);432                    this.fsw._emit(EV.CHANGE, path, entry.stats);433                }434            }435            else {436                dir.add(item);437                this.fsw._symlinkPaths.set(full, linkPath);438                this.fsw._emit(EV.ADD, path, entry.stats);439            }440            this.fsw._emitReady();441            return true;442        }443        // don't follow the same symlink more than once444        if (this.fsw._symlinkPaths.has(full)) {445            return true;446        }447        this.fsw._symlinkPaths.set(full, true);448    }449    _handleRead(directory, initialAdd, wh, target, dir, depth, throttler) {450        // Normalize the directory name on Windows451        directory = sp.join(directory, '');452        const throttleKey = target ? `${directory}:${target}` : directory;453        throttler = this.fsw._throttle('readdir', throttleKey, 1000);454        if (!throttler)455            return;456        const previous = this.fsw._getWatchedDir(wh.path);457        const current = new Set();458        let stream = this.fsw._readdirp(directory, {459            fileFilter: (entry) => wh.filterPath(entry),460            directoryFilter: (entry) => wh.filterDir(entry),461        });462        if (!stream)463            return;464        stream465            .on(STR_DATA, async (entry) => {466            if (this.fsw.closed) {467                stream = undefined;468                return;469            }470            const item = entry.path;471            let path = sp.join(directory, item);472            current.add(item);473            if (entry.stats.isSymbolicLink() &&474                (await this._handleSymlink(entry, directory, path, item))) {475                return;476            }477            if (this.fsw.closed) {478                stream = undefined;479                return;480            }481            // Files that present in current directory snapshot482            // but absent in previous are added to watch list and483            // emit `add` event.484            if (item === target || (!target && !previous.has(item))) {485                this.fsw._incrReadyCount();486                // ensure relativeness of path is preserved in case of watcher reuse487                path = sp.join(dir, sp.relative(dir, path));488                this._addToNodeFs(path, initialAdd, wh, depth + 1);489            }490        })491            .on(EV.ERROR, this._boundHandleError);492        return new Promise((resolve, reject) => {493            if (!stream)494                return reject();495            stream.once(STR_END, () => {496                if (this.fsw.closed) {497                    stream = undefined;498                    return;499                }500                const wasThrottled = throttler ? throttler.clear() : false;501                resolve(undefined);502                // Files that absent in current directory snapshot503                // but present in previous emit `remove` event504                // and are removed from @watched[directory].505                previous506                    .getChildren()507                    .filter((item) => {508                    return item !== directory && !current.has(item);509                })510                    .forEach((item) => {511                    this.fsw._remove(directory, item);512                });513                stream = undefined;514                // one more time for any missed in case changes came in extremely quickly515                if (wasThrottled)516                    this._handleRead(directory, false, wh, target, dir, depth, throttler);517            });518        });519    }520    /**521     * Read directory to add / remove files from `@watched` list and re-read it on change.522     * @param dir fs path523     * @param stats524     * @param initialAdd525     * @param depth relative to user-supplied path526     * @param target child path targeted for watch527     * @param wh Common watch helpers for this path528     * @param realpath529     * @returns closer for the watcher instance.530     */531    async _handleDir(dir, stats, initialAdd, depth, target, wh, realpath) {532        const parentDir = this.fsw._getWatchedDir(sp.dirname(dir));533        const tracked = parentDir.has(sp.basename(dir));534        if (!(initialAdd && this.fsw.options.ignoreInitial) && !target && !tracked) {535            this.fsw._emit(EV.ADD_DIR, dir, stats);536        }537        // ensure dir is tracked (harmless if redundant)538        parentDir.add(sp.basename(dir));539        this.fsw._getWatchedDir(dir);540        let throttler;541        let closer;542        const oDepth = this.fsw.options.depth;543        if ((oDepth == null || depth <= oDepth) && !this.fsw._symlinkPaths.has(realpath)) {544            if (!target) {545                await this._handleRead(dir, initialAdd, wh, target, dir, depth, throttler);546                if (this.fsw.closed)547                    return;548            }549            closer = this._watchWithNodeFs(dir, (dirPath, stats) => {550                // if current directory is removed, do nothing551                if (stats && stats.mtimeMs === 0)552                    return;553                this._handleRead(dirPath, false, wh, target, dir, depth, throttler);554            });555        }556        return closer;557    }558    /**559     * Handle added file, directory, or glob pattern.560     * Delegates call to _handleFile / _handleDir after checks.561     * @param path to file or ir562     * @param initialAdd was the file added at watch instantiation?563     * @param priorWh depth relative to user-supplied path564     * @param depth Child path actually targeted for watch565     * @param target Child path actually targeted for watch566     */567    async _addToNodeFs(path, initialAdd, priorWh, depth, target) {568        const ready = this.fsw._emitReady;569        if (this.fsw._isIgnored(path) || this.fsw.closed) {570            ready();571            return false;572        }573        const wh = this.fsw._getWatchHelpers(path);574        if (priorWh) {575            wh.filterPath = (entry) => priorWh.filterPath(entry);576            wh.filterDir = (entry) => priorWh.filterDir(entry);577        }578        // evaluate what is at the path we're being asked to watch579        try {580            const stats = await statMethods[wh.statMethod](wh.watchPath);581            if (this.fsw.closed)582                return;583            if (this.fsw._isIgnored(wh.watchPath, stats)) {584                ready();585                return false;586            }587            const follow = this.fsw.options.followSymlinks;588            let closer;589            if (stats.isDirectory()) {590                const absPath = sp.resolve(path);591                const targetPath = follow ? await fsrealpath(path) : path;592                if (this.fsw.closed)593                    return;594                closer = await this._handleDir(wh.watchPath, stats, initialAdd, depth, target, wh, targetPath);595                if (this.fsw.closed)596                    return;597                // preserve this symlink's target path598                if (absPath !== targetPath && targetPath !== undefined) {599                    this.fsw._symlinkPaths.set(absPath, targetPath);600                }601            }602            else if (stats.isSymbolicLink()) {603                const targetPath = follow ? await fsrealpath(path) : path;604                if (this.fsw.closed)605                    return;606                const parent = sp.dirname(wh.watchPath);607                this.fsw._getWatchedDir(parent).add(wh.watchPath);608                this.fsw._emit(EV.ADD, wh.watchPath, stats);609                closer = await this._handleDir(parent, stats, initialAdd, depth, path, wh, targetPath);610                if (this.fsw.closed)611                    return;612                // preserve this symlink's target path613                if (targetPath !== undefined) {614                    this.fsw._symlinkPaths.set(sp.resolve(path), targetPath);615                }616            }617            else {618                closer = this._handleFile(wh.watchPath, stats, initialAdd);619            }620            ready();621            if (closer)622                this.fsw._addPathCloser(path, closer);623            return false;624        }625        catch (error) {626            if (this.fsw._handleError(error)) {627                ready();628                return path;629            }630        }631    }632}633