Brunobkr/llama.cpp_AlgMor24_github

1#include "llama-mmap.h"2 3#include "llama-impl.h"4 5#include "ggml.h"6 7#include <cstring>8#include <climits>9#include <stdexcept>10#include <cerrno>11#include <algorithm>12 13#ifdef __has_include14    #if __has_include(<unistd.h>)15        #include <unistd.h>16        #include <fcntl.h>17        #include <sys/stat.h>18        #if defined(_POSIX_MAPPED_FILES)19            #include <sys/mman.h>20        #endif21        #if defined(_POSIX_MEMLOCK_RANGE)22            #include <sys/resource.h>23        #endif24    #endif25#endif26 27#if defined(_WIN32)28    #define WIN32_LEAN_AND_MEAN29    #ifndef NOMINMAX30        #define NOMINMAX31    #endif32    #include <windows.h>33    #ifndef PATH_MAX34        #define PATH_MAX MAX_PATH35    #endif36    #include <io.h>37#endif38 39#if defined(__APPLE__)40#include <TargetConditionals.h>41#endif42 43#ifdef _WIN3244#    define llama_mmap_ftell _ftelli6445#    define llama_mmap_fseek _fseeki6446#else47#    define llama_mmap_ftell ftello48#    define llama_mmap_fseek fseeko49#endif50 51// TODO: consider moving to llama-impl.h if needed in more places52#if defined(_WIN32)53static std::string llama_format_win_err(DWORD err) {54    LPSTR buf;55    size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,56                                 NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);57    if (!size) {58        return "FormatMessageA failed";59    }60    std::string ret(buf, size);61    LocalFree(buf);62    return ret;63}64#endif65 66// llama_file67 68struct llama_file::impl {69#if defined(_WIN32)70    HANDLE fp_win32;71    std::string GetErrorMessageWin32(DWORD error_code) const {72        std::string ret;73        LPSTR lpMsgBuf = NULL;74        DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,75                                    NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL);76        if (!bufLen) {77            ret = format("Win32 error code: %lx", error_code);78        } else {79            ret = lpMsgBuf;80            LocalFree(lpMsgBuf);81        }82 83        return ret;84    }85 86    impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) {87        fp = ggml_fopen(fname, mode);88        if (fp == NULL) {89            throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));90        }91        fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));92        seek(0, SEEK_END);93        size = tell();94        seek(0, SEEK_SET);95    }96 97    impl(FILE * file) : owns_fp(false) {98        fp = file;99        fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));100        seek(0, SEEK_END);101        size = tell();102        seek(0, SEEK_SET);103    }104 105    size_t tell() const {106        LARGE_INTEGER li;107        li.QuadPart = 0;108        BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT);109        if (!ret) {110            throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));111        }112 113        return li.QuadPart;114    }115 116    void seek(size_t offset, int whence) const {117        static_assert(SEEK_SET == FILE_BEGIN, "SEEK_SET != FILE_BEGIN");118        static_assert(SEEK_CUR == FILE_CURRENT, "SEEK_CUR != FILE_CURRENT");119        static_assert(SEEK_END == FILE_END, "SEEK_END != FILE_END");120 121        LARGE_INTEGER li;122        li.QuadPart = offset;123        BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence);124        if (!ret) {125            throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));126        }127    }128 129    void read_raw(void * ptr, size_t len) {130        size_t bytes_read = 0;131        while (bytes_read < len) {132            size_t chunk_size = std::min<size_t>(len - bytes_read, 64*1024*1024);133            DWORD chunk_read = 0;134            BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);135            if (!result) {136                throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));137            }138            if (chunk_read < chunk_size || chunk_read == 0) {139                throw std::runtime_error("unexpectedly reached end of file");140            }141 142            bytes_read += chunk_read;143        }144    }145 146    uint32_t read_u32() {147        uint32_t val;148        read_raw(&val, sizeof(val));149        return val;150    }151 152    void write_raw(const void * ptr, size_t len) const {153        size_t bytes_written = 0;154        while (bytes_written < len) {155            size_t chunk_size = std::min<size_t>(len - bytes_written, 64*1024*1024);156            DWORD chunk_written = 0;157            BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL);158            if (!result) {159                throw std::runtime_error(format("write error: %s", GetErrorMessageWin32(GetLastError()).c_str()));160            }161            if (chunk_written < chunk_size || chunk_written == 0) {162                throw std::runtime_error("unexpectedly failed to write bytes");163            }164 165            bytes_written += chunk_written;166        }167    }168 169    void write_u32(uint32_t val) const {170        write_raw(&val, sizeof(val));171    }172 173    bool has_direct_io() const {174        return true;175    }176 177    ~impl() {178        if (fp && owns_fp) {179            std::fclose(fp);180        }181    }182#else183    impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {184#ifdef __linux__185        // Try unbuffered I/O for read only186        if (use_direct_io && std::strcmp(mode, "rb") == 0) {187            if (init_fd()) {188                return;189            }190            LLAMA_LOG_WARN("Failed to open file '%s' with error: %s. Falling back to buffered I/O",191                           fname, strerror(errno));192        }193#endif194        init_fp(mode);195    }196 197#ifdef __linux__198    bool init_fd() {199        fd = open(fname.c_str(), O_RDONLY | O_DIRECT);200 201        if (fd != -1) {202            struct stat file_stats{};203            fstat(fd, &file_stats);204 205            size = file_stats.st_size;206            alignment = file_stats.st_blksize;207 208            off_t ret = lseek(fd, 0, SEEK_SET);209            if (ret == -1) {210                throw std::runtime_error(format("seek error: %s", strerror(errno)));211            }212            return true;213        }214        return false;215    }216#endif217 218    void init_fp(const char * mode) {219        fp = ggml_fopen(fname.c_str(), mode);220        if (fp == NULL) {221            throw std::runtime_error(format("failed to open %s: %s", fname.c_str(), strerror(errno)));222        }223        seek(0, SEEK_END);224        size = tell();225        seek(0, SEEK_SET);226    }227 228    impl(FILE * file) : fname("(file*)"), owns_fp(false) {229        fp = file;230        seek(0, SEEK_END);231        size = tell();232        seek(0, SEEK_SET);233    }234 235    size_t tell() const {236        if (fd == -1) {237            off_t ret = llama_mmap_ftell(fp);238            if (ret == -1) {239                throw std::runtime_error(format("ftell error: %s", strerror(errno)));240            }241 242            return (size_t) ret;243        }244 245        off_t pos = lseek(fd, 0, SEEK_CUR);246        if (pos == -1) {247            throw std::runtime_error(format("lseek error: %s", strerror(errno)));248        }249        return (size_t) pos;250    }251 252    void seek(size_t offset, int whence) const {253        off_t ret = 0;254        if (fd == -1) {255            ret = llama_mmap_fseek(fp, offset, whence);256        } else {257            ret = lseek(fd, offset, whence);258        }259        if (ret == -1) {260            throw std::runtime_error(format("seek error: %s", strerror(errno)));261        }262    }263 264    void read_raw_unsafe(void * ptr, size_t len) {265        if (len == 0) {266            return;267        }268        errno = 0;269        if (fd == -1) {270            const size_t curr_off = tell();271            const size_t to_read = std::min(len, size - curr_off);272 273            std::size_t ret = std::fread(ptr, to_read, 1, fp);274            if (ferror(fp)) {275                throw std::runtime_error(format("read error: %s", strerror(errno)));276            }277            if (to_read > 0 && ret != 1) {278                throw std::runtime_error("unexpectedly reached end of file");279            }280        } else {281            size_t bytes_read = 0;282            while (bytes_read < len) {283                const size_t to_read = len - bytes_read;284                ssize_t ret = ::read(fd, reinterpret_cast<char *>(ptr) + bytes_read, to_read);285 286                if (ret == -1) {287                    if (errno == EINTR) {288                        continue;  // Interrupted by signal, retry289                    }290                    // Fallback to std::fread in case the DMA controller cannot access the buffer291                    if (errno == EFAULT || errno == EINVAL) {292                        LLAMA_LOG_WARN("%s: Falling back to buffered IO due to %s\n", __func__, strerror(errno));293                        auto curr_off = tell();294                        close(fd);295                        fd = -1;296                        alignment = 1;297                        init_fp("rb");298                        seek(curr_off, SEEK_SET);299                        read_raw_unsafe(ptr, len);300                        return;301                    }302                    throw std::runtime_error(format("read error: %s", strerror(errno)));303                }304                if (ret == 0) {305                    // EOF: allow if this read was only pulling alignment padding past file end306                    off_t pos = lseek(fd, 0, SEEK_CUR);307                    if (pos != -1 && (size_t) pos == size) {308                        std::memset(reinterpret_cast<char *>(ptr) + bytes_read, 0, len - bytes_read);309                        return;310                    }311                    throw std::runtime_error("unexpectedly reached end of file");312                }313 314                bytes_read += (size_t) ret;315            }316        }317    }318 319    void read_aligned_chunk(void * dest, size_t size) {320        size_t offset = tell();321        off_t aligned_offset = offset & ~(alignment - 1);322        off_t offset_from_alignment = offset - aligned_offset;323        size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1);324 325        void * raw_buffer = nullptr;326        int ret = posix_memalign(&raw_buffer, alignment, bytes_to_read);327        if (ret != 0) {328            throw std::runtime_error(format("posix_memalign failed with error %d", ret));329        }330 331        struct aligned_buffer_deleter {332            void operator()(void * p) const { free(p); }333        };334        std::unique_ptr<void, aligned_buffer_deleter> buffer(raw_buffer);335 336        seek(aligned_offset, SEEK_SET);337        read_raw_unsafe(buffer.get(), bytes_to_read);338 339        uintptr_t actual_data = reinterpret_cast<uintptr_t>(buffer.get()) + offset_from_alignment;340        memcpy(dest, reinterpret_cast<void *>(actual_data), size);341    }342 343    void read_raw(void * ptr, size_t len) {344        if (has_direct_io()) {345            read_aligned_chunk(ptr, len);346        } else {347            read_raw_unsafe(ptr, len);348        }349    }350 351    uint32_t read_u32() {352        uint32_t ret;353        read_raw(&ret, sizeof(ret));354        return ret;355    }356 357    void write_raw(const void * ptr, size_t len) const {358        if (len == 0) {359            return;360        }361        errno = 0;362        size_t ret = std::fwrite(ptr, len, 1, fp);363        if (ret != 1) {364            throw std::runtime_error(format("write error: %s", strerror(errno)));365        }366    }367 368    void write_u32(uint32_t val) const {369        write_raw(&val, sizeof(val));370    }371 372    bool has_direct_io() const {373        return fd != -1 && alignment > 1;374    }375 376    ~impl() {377        if (fd != -1) {378            close(fd);379        } else if (owns_fp) {380            std::fclose(fp);381        }382    }383    int fd = -1;384    std::string fname;385#endif386 387    size_t read_alignment() const {388        return alignment;389    }390 391    size_t alignment = 1;392 393    FILE * fp{};394    size_t size{};395    bool owns_fp = true;396};397 398llama_file::llama_file(const char * fname, const char * mode, const bool use_direct_io) :399    pimpl(std::make_unique<impl>(fname, mode, use_direct_io)) {}400 401llama_file::llama_file(FILE * file) : pimpl(std::make_unique<impl>(file)) {}402 403llama_file::~llama_file() = default;404 405size_t llama_file::tell() const { return pimpl->tell(); }406size_t llama_file::size() const { return pimpl->size; }407 408size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }409bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }410 411int llama_file::file_id() const {412#ifdef _WIN32413    return _fileno(pimpl->fp);414#else415    if (pimpl->fd != -1) {416        return pimpl->fd;417    }418#if defined(fileno)419    return fileno(pimpl->fp);420#else421    return ::fileno(pimpl->fp);422#endif423#endif424}425 426void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); }427void llama_file::read_raw(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }428#ifdef _WIN32429void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }430#else431void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw_unsafe(ptr, len); }432#endif433 434uint32_t llama_file::read_u32() { return pimpl->read_u32(); }435 436void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }437void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }438 439// llama_mmap440 441struct llama_mmap::impl {442#ifdef _POSIX_MAPPED_FILES443    std::vector<std::pair<size_t, size_t>> mapped_fragments;444 445    impl(struct llama_file * file, size_t prefetch, bool numa) {446        size = file->size();447        int fd = file->file_id();448        int flags = MAP_SHARED;449        if (numa) { prefetch = 0; }450#ifdef __linux__451        if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {452            LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",453                    strerror(errno));454        }455        if (prefetch) { flags |= MAP_POPULATE; }456#endif457        addr = mmap(NULL, file->size(), PROT_READ, flags, fd, 0);458        if (addr == MAP_FAILED) {459            throw std::runtime_error(format("mmap failed: %s", strerror(errno)));460        }461 462        if (prefetch > 0) {463            if (posix_madvise(addr, std::min(file->size(), prefetch), POSIX_MADV_WILLNEED)) {464                LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",465                        strerror(errno));466            }467        }468        if (numa) {469            if (posix_madvise(addr, file->size(), POSIX_MADV_RANDOM)) {470                LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",471                        strerror(errno));472            }473        }474 475        mapped_fragments.emplace_back(0, file->size());476    }477 478    static void align_range(size_t * first, size_t * last, size_t page_size) {479        size_t offset_in_page = *first & (page_size - 1);480        size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;481        *first += offset_to_page;482 483        *last = *last & ~(page_size - 1);484 485        if (*last <= *first) {486            *last = *first;487        }488    }489 490    void unmap_fragment(size_t first, size_t last) {491        int page_size = sysconf(_SC_PAGESIZE);492        align_range(&first, &last, page_size);493        size_t len = last - first;494 495        if (len == 0) {496            return;497        }498 499        GGML_ASSERT(first % page_size == 0);500        GGML_ASSERT(last % page_size == 0);501        GGML_ASSERT(last > first);502 503        void * next_page_start = (uint8_t *) addr + first;504 505        if (munmap(next_page_start, len)) {506            LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));507        }508 509        std::vector<std::pair<size_t, size_t>> new_mapped_fragments;510        for (const auto & frag : mapped_fragments) {511            if (frag.first < first && frag.second > last) {512                new_mapped_fragments.emplace_back(frag.first, first);513                new_mapped_fragments.emplace_back(last, frag.second);514            } else if (frag.first < first && frag.second > first) {515                new_mapped_fragments.emplace_back(frag.first, first);516            } else if (frag.first < last && frag.second > last) {517                new_mapped_fragments.emplace_back(last, frag.second);518            } else if (frag.first >= first && frag.second <= last) {519            } else {520                new_mapped_fragments.push_back(frag);521            }522        }523        mapped_fragments = std::move(new_mapped_fragments);524    }525 526    ~impl() {527        for (const auto & frag : mapped_fragments) {528            if (munmap((char *) addr + frag.first, frag.second - frag.first)) {529                LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));530            }531        }532    }533#elif defined(_WIN32)534    HANDLE hMapping = nullptr;535 536    impl(struct llama_file * file, size_t prefetch, bool numa) {537        GGML_UNUSED(numa);538 539        size = file->size();540 541        HANDLE hFile = (HANDLE) _get_osfhandle(file->file_id());542 543        hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);544 545        if (hMapping == NULL) {546            DWORD error = GetLastError();547            throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));548        }549 550        addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);551        DWORD error = GetLastError();552 553        if (addr == NULL) {554            CloseHandle(hMapping);555            throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));556        }557 558        if (prefetch > 0) {559#if _WIN32_WINNT >= 0x602560            BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);561            HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");562 563            pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");564 565            if (pPrefetchVirtualMemory) {566                WIN32_MEMORY_RANGE_ENTRY range;567                range.VirtualAddress = addr;568                range.NumberOfBytes = (SIZE_T) std::min(size, prefetch);569                if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {570                    LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",571                            llama_format_win_err(GetLastError()).c_str());572                }573            }574#else575            LLAMA_LOG_DEBUG("skipping PrefetchVirtualMemory because _WIN32_WINNT < 0x602\n");576#endif577        }578    }579 580    void unmap_fragment(size_t first, size_t last) {581        GGML_UNUSED(first);582        GGML_UNUSED(last);583    }584 585    ~impl() {586        if (hMapping) {587            if (addr) {588                if (!UnmapViewOfFile(addr)) {589                    LLAMA_LOG_WARN("warning: UnmapViewOfFile failed: %s\n",590                            llama_format_win_err(GetLastError()).c_str());591                }592            }593            if (!CloseHandle(hMapping)) {594                LLAMA_LOG_WARN("warning: CloseHandle failed: %s\n",595                        llama_format_win_err(GetLastError()).c_str());596            }597        }598    }599#else600    impl(struct llama_file * file, size_t prefetch, bool numa) {601        GGML_UNUSED(file);602        GGML_UNUSED(prefetch);603        GGML_UNUSED(numa);604 605        throw std::runtime_error("mmap not supported");606    }607 608    void unmap_fragment(size_t first, size_t last) {609        GGML_UNUSED(first);610        GGML_UNUSED(last);611 612        throw std::runtime_error("mmap not supported");613    }614#endif615 616    void * addr;617    size_t size;618};619 620llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique<impl>(file, prefetch, numa)) {}621llama_mmap::~llama_mmap() = default;622 623size_t llama_mmap::size() const { return pimpl->size; }624void * llama_mmap::addr() const { return pimpl->addr; }625 626void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); }627 628#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)629const bool llama_mmap::SUPPORTED  = true;630#else631const bool llama_mmap::SUPPORTED  = false;632#endif633 634// llama_mlock635 636struct llama_mlock::impl {637#ifdef _POSIX_MEMLOCK_RANGE638    static size_t lock_granularity() {639        return (size_t) sysconf(_SC_PAGESIZE);640    }641 642    bool raw_lock(const void * addr, size_t size) const {643        if (!mlock(addr, size)) {644            return true;645        }646 647#ifdef __APPLE__648#define MLOCK_SUGGESTION \649        "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \650        "decreasing 'vm.global_no_user_wire_amount'.  Also try increasing RLIMIT_MEMLOCK (ulimit -l).\n"651#else652#define MLOCK_SUGGESTION \653        "Try increasing RLIMIT_MEMLOCK ('ulimit -l' as root).\n"654#endif655 656        char* errmsg = std::strerror(errno);657        bool suggest = (errno == ENOMEM);658#if defined(TARGET_OS_VISION) || defined(TARGET_OS_TV) || defined(_AIX) || defined(__HAIKU__)659        // visionOS/tvOS/Haiku don't support RLIMIT_MEMLOCK660        // Skip resource limit checks on these platforms661        suggest = false;662#else663        struct rlimit lock_limit;664        if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) {665            suggest = false;666        }667        if (suggest && ((uint64_t)lock_limit.rlim_max > (uint64_t)lock_limit.rlim_cur + size)) {668            suggest = false;669        }670#endif671 672        LLAMA_LOG_WARN("warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",673                size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");674        return false;675    }676 677    static void raw_unlock(void * addr, size_t size) {678        if (munlock(addr, size)) {679            LLAMA_LOG_WARN("warning: failed to munlock buffer: %s\n", std::strerror(errno));680        }681    }682#elif defined(_WIN32)683    static size_t lock_granularity() {684        SYSTEM_INFO si;685        GetSystemInfo(&si);686        return (size_t) si.dwPageSize;687    }688 689    bool raw_lock(void * ptr, size_t len) const {690        for (int tries = 1; ; tries++) {691            if (VirtualLock(ptr, len)) {692                return true;693            }694            if (tries == 2) {695                LLAMA_LOG_WARN("warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",696                    len, size, llama_format_win_err(GetLastError()).c_str());697                return false;698            }699 700            SIZE_T min_ws_size, max_ws_size;701            if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {702                LLAMA_LOG_WARN("warning: GetProcessWorkingSetSize failed: %s\n",703                        llama_format_win_err(GetLastError()).c_str());704                return false;705            }706            size_t increment = len + 1048576;707            min_ws_size += increment;708            max_ws_size += increment;709            if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {710                LLAMA_LOG_WARN("warning: SetProcessWorkingSetSize failed: %s\n",711                        llama_format_win_err(GetLastError()).c_str());712                return false;713            }714        }715    }716 717    static void raw_unlock(void * ptr, size_t len) {718        if (!VirtualUnlock(ptr, len)) {719            LLAMA_LOG_WARN("warning: failed to VirtualUnlock buffer: %s\n",720                    llama_format_win_err(GetLastError()).c_str());721        }722    }723#else724    static size_t lock_granularity() {725        return (size_t) 65536;726    }727 728    bool raw_lock(const void * addr, size_t len) const {729        LLAMA_LOG_WARN("warning: mlock not supported on this system\n");730        return false;731    }732 733    static void raw_unlock(const void * addr, size_t len) {}734#endif735 736    impl() : addr(NULL), size(0), failed_already(false) {}737 738    void init(void * ptr) {739        GGML_ASSERT(addr == NULL && size == 0);740        addr = ptr;741    }742 743    void grow_to(size_t target_size) {744        GGML_ASSERT(addr);745        if (failed_already) {746            return;747        }748        size_t granularity = lock_granularity();749        target_size = (target_size + granularity - 1) & ~(granularity - 1);750        if (target_size > size) {751            if (raw_lock((uint8_t *) addr + size, target_size - size)) {752                size = target_size;753            } else {754                failed_already = true;755            }756        }757    }758 759    void * addr;760    size_t size;761 762    bool failed_already;763};764 765llama_mlock::llama_mlock() : pimpl(std::make_unique<impl>()) {}766llama_mlock::~llama_mlock() = default;767 768void llama_mlock::init(void * ptr) { pimpl->init(ptr); }769void llama_mlock::grow_to(size_t target_size) { pimpl->grow_to(target_size); }770 771#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)772const bool llama_mlock::SUPPORTED = true;773#else774const bool llama_mlock::SUPPORTED = false;775#endif776 777size_t llama_path_max() {778    return PATH_MAX;779}780