Brunobkr/llama.cpp_AlgMor24_github

1import { lstat, readdir, realpath, stat } from 'node:fs/promises';2import { join as pjoin, relative as prelative, resolve as presolve, sep as psep } from 'node:path';3import { Readable } from 'node:stream';4export const EntryTypes = {5    FILE_TYPE: 'files',6    DIR_TYPE: 'directories',7    FILE_DIR_TYPE: 'files_directories',8    EVERYTHING_TYPE: 'all',9};10const defaultOptions = {11    root: '.',12    fileFilter: (_entryInfo) => true,13    directoryFilter: (_entryInfo) => true,14    type: EntryTypes.FILE_TYPE,15    lstat: false,16    depth: 2147483648,17    alwaysStat: false,18    highWaterMark: 4096,19};20Object.freeze(defaultOptions);21const RECURSIVE_ERROR_CODE = 'READDIRP_RECURSIVE_ERROR';22const NORMAL_FLOW_ERRORS = new Set(['ENOENT', 'EPERM', 'EACCES', 'ELOOP', RECURSIVE_ERROR_CODE]);23const ALL_TYPES = [24    EntryTypes.DIR_TYPE,25    EntryTypes.EVERYTHING_TYPE,26    EntryTypes.FILE_DIR_TYPE,27    EntryTypes.FILE_TYPE,28];29const DIR_TYPES = new Set([30    EntryTypes.DIR_TYPE,31    EntryTypes.EVERYTHING_TYPE,32    EntryTypes.FILE_DIR_TYPE,33]);34const FILE_TYPES = new Set([35    EntryTypes.EVERYTHING_TYPE,36    EntryTypes.FILE_DIR_TYPE,37    EntryTypes.FILE_TYPE,38]);39const isNormalFlowError = (error) => NORMAL_FLOW_ERRORS.has(error.code);40const wantBigintFsStats = process.platform === 'win32';41const emptyFn = (_entryInfo) => true;42const normalizeFilter = (filter) => {43    if (filter === undefined)44        return emptyFn;45    if (typeof filter === 'function')46        return filter;47    if (typeof filter === 'string') {48        const fl = filter.trim();49        return (entry) => entry.basename === fl;50    }51    if (Array.isArray(filter)) {52        const trItems = filter.map((item) => item.trim());53        return (entry) => trItems.some((f) => entry.basename === f);54    }55    return emptyFn;56};57/** Readable readdir stream, emitting new files as they're being listed. */58export class ReaddirpStream extends Readable {59    parents;60    reading;61    parent;62    _stat;63    _maxDepth;64    _wantsDir;65    _wantsFile;66    _wantsEverything;67    _root;68    _isDirent;69    _statsProp;70    _rdOptions;71    _fileFilter;72    _directoryFilter;73    constructor(options = {}) {74        super({75            objectMode: true,76            autoDestroy: true,77            highWaterMark: options.highWaterMark,78        });79        const opts = { ...defaultOptions, ...options };80        const { root, type } = opts;81        this._fileFilter = normalizeFilter(opts.fileFilter);82        this._directoryFilter = normalizeFilter(opts.directoryFilter);83        const statMethod = opts.lstat ? lstat : stat;84        // Use bigint stats if it's windows and stat() supports options (node 10+).85        if (wantBigintFsStats) {86            this._stat = (path) => statMethod(path, { bigint: true });87        }88        else {89            this._stat = statMethod;90        }91        this._maxDepth =92            opts.depth != null && Number.isSafeInteger(opts.depth) ? opts.depth : defaultOptions.depth;93        this._wantsDir = type ? DIR_TYPES.has(type) : false;94        this._wantsFile = type ? FILE_TYPES.has(type) : false;95        this._wantsEverything = type === EntryTypes.EVERYTHING_TYPE;96        this._root = presolve(root);97        this._isDirent = !opts.alwaysStat;98        this._statsProp = this._isDirent ? 'dirent' : 'stats';99        this._rdOptions = { encoding: 'utf8', withFileTypes: this._isDirent };100        // Launch stream with one parent, the root dir.101        this.parents = [this._exploreDir(root, 1)];102        this.reading = false;103        this.parent = undefined;104    }105    async _read(batch) {106        if (this.reading)107            return;108        this.reading = true;109        try {110            while (!this.destroyed && batch > 0) {111                const par = this.parent;112                const fil = par && par.files;113                if (fil && fil.length > 0) {114                    const { path, depth } = par;115                    const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path));116                    const awaited = await Promise.all(slice);117                    for (const entry of awaited) {118                        if (!entry)119                            continue;120                        if (this.destroyed)121                            return;122                        const entryType = await this._getEntryType(entry);123                        if (entryType === 'directory' && this._directoryFilter(entry)) {124                            if (depth <= this._maxDepth) {125                                this.parents.push(this._exploreDir(entry.fullPath, depth + 1));126                            }127                            if (this._wantsDir) {128                                this.push(entry);129                                batch--;130                            }131                        }132                        else if ((entryType === 'file' || this._includeAsFile(entry)) &&133                            this._fileFilter(entry)) {134                            if (this._wantsFile) {135                                this.push(entry);136                                batch--;137                            }138                        }139                    }140                }141                else {142                    const parent = this.parents.pop();143                    if (!parent) {144                        this.push(null);145                        break;146                    }147                    this.parent = await parent;148                    if (this.destroyed)149                        return;150                }151            }152        }153        catch (error) {154            this.destroy(error);155        }156        finally {157            this.reading = false;158        }159    }160    async _exploreDir(path, depth) {161        let files;162        try {163            files = await readdir(path, this._rdOptions);164        }165        catch (error) {166            this._onError(error);167        }168        return { files, depth, path };169    }170    async _formatEntry(dirent, path) {171        let entry;172        const basename = this._isDirent ? dirent.name : dirent;173        try {174            const fullPath = presolve(pjoin(path, basename));175            entry = { path: prelative(this._root, fullPath), fullPath, basename };176            entry[this._statsProp] = this._isDirent ? dirent : await this._stat(fullPath);177        }178        catch (err) {179            this._onError(err);180            return;181        }182        return entry;183    }184    _onError(err) {185        if (isNormalFlowError(err) && !this.destroyed) {186            this.emit('warn', err);187        }188        else {189            this.destroy(err);190        }191    }192    async _getEntryType(entry) {193        // entry may be undefined, because a warning or an error were emitted194        // and the statsProp is undefined195        if (!entry && this._statsProp in entry) {196            return '';197        }198        const stats = entry[this._statsProp];199        if (stats.isFile())200            return 'file';201        if (stats.isDirectory())202            return 'directory';203        if (stats && stats.isSymbolicLink()) {204            const full = entry.fullPath;205            try {206                const entryRealPath = await realpath(full);207                const entryRealPathStats = await lstat(entryRealPath);208                if (entryRealPathStats.isFile()) {209                    return 'file';210                }211                if (entryRealPathStats.isDirectory()) {212                    const len = entryRealPath.length;213                    if (full.startsWith(entryRealPath) && full.substr(len, 1) === psep) {214                        const recursiveError = new Error(`Circular symlink detected: "${full}" points to "${entryRealPath}"`);215                        // @ts-ignore216                        recursiveError.code = RECURSIVE_ERROR_CODE;217                        return this._onError(recursiveError);218                    }219                    return 'directory';220                }221            }222            catch (error) {223                this._onError(error);224                return '';225            }226        }227    }228    _includeAsFile(entry) {229        const stats = entry && entry[this._statsProp];230        return stats && this._wantsEverything && !stats.isDirectory();231    }232}233/**234 * Streaming version: Reads all files and directories in given root recursively.235 * Consumes ~constant small amount of RAM.236 * @param root Root directory237 * @param options Options to specify root (start directory), filters and recursion depth238 */239export function readdirp(root, options = {}) {240    // @ts-ignore241    let type = options.entryType || options.type;242    if (type === 'both')243        type = EntryTypes.FILE_DIR_TYPE; // backwards-compatibility244    if (type)245        options.type = type;246    if (!root) {247        throw new Error('readdirp: root argument is required. Usage: readdirp(root, options)');248    }249    else if (typeof root !== 'string') {250        throw new TypeError('readdirp: root argument must be a string. Usage: readdirp(root, options)');251    }252    else if (type && !ALL_TYPES.includes(type)) {253        throw new Error(`readdirp: Invalid type passed. Use one of ${ALL_TYPES.join(', ')}`);254    }255    options.root = root;256    return new ReaddirpStream(options);257}258/**259 * Promise version: Reads all files and directories in given root recursively.260 * Compared to streaming version, will consume a lot of RAM e.g. when 1 million files are listed.261 * @returns array of paths and their entry infos262 */263export function readdirpPromise(root, options = {}) {264    return new Promise((resolve, reject) => {265        const files = [];266        readdirp(root, options)267            .on('data', (entry) => files.push(entry))268            .on('end', () => resolve(files))269            .on('error', (error) => reject(error));270    });271}272export default readdirp;273