echodict/llama.cpp

1#include "llama-mmap.h"2 3#include "llama-impl.h"4 5#include "ggml.h"6 7#include <cstring>8#include <climits>9#include <stdexcept>10#include <cerrno>11#include <algorithm>12 13#ifdef __has_include14    #if __has_include(<unistd.h>)15        #include <unistd.h>16        #include <fcntl.h>17        #include <sys/stat.h>18        #if defined(_POSIX_MAPPED_FILES)19            #include <sys/mman.h>20        #endif21        #if defined(_POSIX_MEMLOCK_RANGE)22            #include <sys/resource.h>23        #endif24    #endif25#endif26 27#if defined(_WIN32)28    #define WIN32_LEAN_AND_MEAN29    #ifndef NOMINMAX30        #define NOMINMAX31    #endif32    #include <windows.h>33    #ifndef PATH_MAX34        #define PATH_MAX MAX_PATH35    #endif36    #include <io.h>37#endif38 39#if defined(__APPLE__)40#include <TargetConditionals.h>41#endif42 43// TODO: consider moving to llama-impl.h if needed in more places44#if defined(_WIN32)45static std::string llama_format_win_err(DWORD err) {46    LPSTR buf;47    size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,48                                 NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);49    if (!size) {50        return "FormatMessageA failed";51    }52    std::string ret(buf, size);53    LocalFree(buf);54    return ret;55}56#endif57 58// llama_file59 60struct llama_file::impl {61#if defined(_WIN32)62    HANDLE fp_win32;63    std::string GetErrorMessageWin32(DWORD error_code) const {64        std::string ret;65        LPSTR lpMsgBuf = NULL;66        DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,67                                    NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL);68        if (!bufLen) {69            ret = format("Win32 error code: %lx", error_code);70        } else {71            ret = lpMsgBuf;72            LocalFree(lpMsgBuf);73        }74 75        return ret;76    }77 78    impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) {79        fp = ggml_fopen(fname, mode);80        if (fp == NULL) {81            throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));82        }83        fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));84        seek(0, SEEK_END);85        size = tell();86        seek(0, SEEK_SET);87    }88 89    impl(FILE * file) : owns_fp(false) {90        fp = file;91        fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));92        seek(0, SEEK_END);93        size = tell();94        seek(0, SEEK_SET);95    }96 97    size_t tell() const {98        LARGE_INTEGER li;99        li.QuadPart = 0;100        BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT);101        if (!ret) {102            throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));103        }104 105        return li.QuadPart;106    }107 108    void seek(size_t offset, int whence) const {109        static_assert(SEEK_SET == FILE_BEGIN, "SEEK_SET != FILE_BEGIN");110        static_assert(SEEK_CUR == FILE_CURRENT, "SEEK_CUR != FILE_CURRENT");111        static_assert(SEEK_END == FILE_END, "SEEK_END != FILE_END");112 113        LARGE_INTEGER li;114        li.QuadPart = offset;115        BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence);116        if (!ret) {117            throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));118        }119    }120 121    void read_raw(void * ptr, size_t len) {122        size_t bytes_read = 0;123        while (bytes_read < len) {124            size_t chunk_size = std::min<size_t>(len - bytes_read, 64*1024*1024);125            DWORD chunk_read = 0;126            BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);127            if (!result) {128                throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));129            }130            if (chunk_read < chunk_size || chunk_read == 0) {131                throw std::runtime_error("unexpectedly reached end of file");132            }133 134            bytes_read += chunk_read;135        }136    }137 138    uint32_t read_u32() {139        uint32_t val;140        read_raw(&val, sizeof(val));141        return val;142    }143 144    void write_raw(const void * ptr, size_t len) const {145        size_t bytes_written = 0;146        while (bytes_written < len) {147            size_t chunk_size = std::min<size_t>(len - bytes_written, 64*1024*1024);148            DWORD chunk_written = 0;149            BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL);150            if (!result) {151                throw std::runtime_error(format("write error: %s", GetErrorMessageWin32(GetLastError()).c_str()));152            }153            if (chunk_written < chunk_size || chunk_written == 0) {154                throw std::runtime_error("unexpectedly failed to write bytes");155            }156 157            bytes_written += chunk_written;158        }159    }160 161    void write_u32(uint32_t val) const {162        write_raw(&val, sizeof(val));163    }164 165    bool has_direct_io() const {166        return true;167    }168 169    ~impl() {170        if (fp && owns_fp) {171            std::fclose(fp);172        }173    }174#else175    impl(const char * fname, const char * mode, [[maybe_unused]] const bool use_direct_io = false) : fname(fname) {176#ifdef __linux__177        // Try unbuffered I/O for read only178        if (use_direct_io && std::strcmp(mode, "rb") == 0) {179            if (init_fd()) {180                return;181            }182            LLAMA_LOG_WARN("Failed to open file '%s' with error: %s. Falling back to buffered I/O",183                           fname, strerror(errno));184        }185#endif186        init_fp(mode);187    }188 189#ifdef __linux__190    bool init_fd() {191        fd = open(fname.c_str(), O_RDONLY | O_DIRECT);192 193        if (fd != -1) {194            struct stat file_stats{};195            fstat(fd, &file_stats);196 197            size = file_stats.st_size;198            alignment = file_stats.st_blksize;199 200            off_t ret = lseek(fd, 0, SEEK_SET);201            if (ret == -1) {202                throw std::runtime_error(format("seek error: %s", strerror(errno)));203            }204            return true;205        }206        return false;207    }208#endif209 210    void init_fp(const char * mode) {211        fp = ggml_fopen(fname.c_str(), mode);212        if (fp == NULL) {213            throw std::runtime_error(format("failed to open %s: %s", fname.c_str(), strerror(errno)));214        }215        seek(0, SEEK_END);216        size = tell();217        seek(0, SEEK_SET);218    }219 220    impl(FILE * file) : fname("(file*)"), owns_fp(false) {221        fp = file;222        seek(0, SEEK_END);223        size = tell();224        seek(0, SEEK_SET);225    }226 227    size_t tell() const {228        if (fd == -1) {229            long ret = std::ftell(fp);230            if (ret == -1) {231                throw std::runtime_error(format("ftell error: %s", strerror(errno)));232            }233 234            return (size_t) ret;235        }236 237        off_t pos = lseek(fd, 0, SEEK_CUR);238        if (pos == -1) {239            throw std::runtime_error(format("lseek error: %s", strerror(errno)));240        }241        return (size_t) pos;242    }243 244    void seek(size_t offset, int whence) const {245        off_t ret = 0;246        if (fd == -1) {247            ret = std::fseek(fp, (long) offset, whence);248        } else {249            ret = lseek(fd, offset, whence);250        }251        if (ret == -1) {252            throw std::runtime_error(format("seek error: %s", strerror(errno)));253        }254    }255 256    void read_raw_unsafe(void * ptr, size_t len) {257        if (len == 0) {258            return;259        }260        errno = 0;261        if (fd == -1) {262            const size_t curr_off = tell();263            const size_t to_read = std::min(len, size - curr_off);264 265            std::size_t ret = std::fread(ptr, to_read, 1, fp);266            if (ferror(fp)) {267                throw std::runtime_error(format("read error: %s", strerror(errno)));268            }269            if (to_read > 0 && ret != 1) {270                throw std::runtime_error("unexpectedly reached end of file");271            }272        } else {273            size_t bytes_read = 0;274            while (bytes_read < len) {275                const size_t to_read = len - bytes_read;276                ssize_t ret = ::read(fd, reinterpret_cast<char *>(ptr) + bytes_read, to_read);277 278                if (ret == -1) {279                    if (errno == EINTR) {280                        continue;  // Interrupted by signal, retry281                    }282                    // Fallback to std::fread in case the DMA controller cannot access the buffer283                    if (errno == EFAULT || errno == EINVAL) {284                        LLAMA_LOG_WARN("%s: Falling back to buffered IO due to %s\n", __func__, strerror(errno));285                        auto curr_off = tell();286                        close(fd);287                        fd = -1;288                        alignment = 1;289                        init_fp("rb");290                        seek(curr_off, SEEK_SET);291                        read_raw_unsafe(ptr, len);292                        return;293                    }294                    throw std::runtime_error(format("read error: %s", strerror(errno)));295                }296                if (ret == 0) {297                    // EOF: allow if this read was only pulling alignment padding past file end298                    off_t pos = lseek(fd, 0, SEEK_CUR);299                    if (pos != -1 && (size_t) pos == size) {300                        std::memset(reinterpret_cast<char *>(ptr) + bytes_read, 0, len - bytes_read);301                        return;302                    }303                    throw std::runtime_error("unexpectedly reached end of file");304                }305 306                bytes_read += (size_t) ret;307            }308        }309    }310 311    void read_aligned_chunk(void * dest, size_t size) {312        size_t offset = tell();313        off_t aligned_offset = offset & ~(alignment - 1);314        off_t offset_from_alignment = offset - aligned_offset;315        size_t bytes_to_read = (offset_from_alignment + size + alignment - 1) & ~(alignment - 1);316 317        void * raw_buffer = nullptr;318        int ret = posix_memalign(&raw_buffer, alignment, bytes_to_read);319        if (ret != 0) {320            throw std::runtime_error(format("posix_memalign failed with error %d", ret));321        }322 323        struct aligned_buffer_deleter {324            void operator()(void * p) const { free(p); }325        };326        std::unique_ptr<void, aligned_buffer_deleter> buffer(raw_buffer);327 328        seek(aligned_offset, SEEK_SET);329        read_raw_unsafe(buffer.get(), bytes_to_read);330 331        uintptr_t actual_data = reinterpret_cast<uintptr_t>(buffer.get()) + offset_from_alignment;332        memcpy(dest, reinterpret_cast<void *>(actual_data), size);333    }334 335    void read_raw(void * ptr, size_t len) {336        if (has_direct_io()) {337            read_aligned_chunk(ptr, len);338        } else {339            read_raw_unsafe(ptr, len);340        }341    }342 343    uint32_t read_u32() {344        uint32_t ret;345        read_raw(&ret, sizeof(ret));346        return ret;347    }348 349    void write_raw(const void * ptr, size_t len) const {350        if (len == 0) {351            return;352        }353        errno = 0;354        size_t ret = std::fwrite(ptr, len, 1, fp);355        if (ret != 1) {356            throw std::runtime_error(format("write error: %s", strerror(errno)));357        }358    }359 360    void write_u32(uint32_t val) const {361        write_raw(&val, sizeof(val));362    }363 364    bool has_direct_io() const {365        return fd != -1 && alignment > 1;366    }367 368    ~impl() {369        if (fd != -1) {370            close(fd);371        } else if (owns_fp) {372            std::fclose(fp);373        }374    }375    int fd = -1;376    std::string fname;377#endif378 379    size_t read_alignment() const {380        return alignment;381    }382 383    size_t alignment = 1;384 385    FILE * fp{};386    size_t size{};387    bool owns_fp = true;388};389 390llama_file::llama_file(const char * fname, const char * mode, const bool use_direct_io) :391    pimpl(std::make_unique<impl>(fname, mode, use_direct_io)) {}392 393llama_file::llama_file(FILE * file) : pimpl(std::make_unique<impl>(file)) {}394 395llama_file::~llama_file() = default;396 397size_t llama_file::tell() const { return pimpl->tell(); }398size_t llama_file::size() const { return pimpl->size; }399 400size_t llama_file::read_alignment() const { return pimpl->read_alignment(); }401bool llama_file::has_direct_io() const { return pimpl->has_direct_io(); }402 403int llama_file::file_id() const {404#ifdef _WIN32405    return _fileno(pimpl->fp);406#else407    if (pimpl->fd != -1) {408        return pimpl->fd;409    }410#if defined(fileno)411    return fileno(pimpl->fp);412#else413    return ::fileno(pimpl->fp);414#endif415#endif416}417 418void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); }419void llama_file::read_raw(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }420#ifdef _WIN32421void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw(ptr, len); }422#else423void llama_file::read_raw_unsafe(void * ptr, size_t len) { pimpl->read_raw_unsafe(ptr, len); }424#endif425 426uint32_t llama_file::read_u32() { return pimpl->read_u32(); }427 428void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }429void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }430 431// llama_mmap432 433struct llama_mmap::impl {434#ifdef _POSIX_MAPPED_FILES435    std::vector<std::pair<size_t, size_t>> mapped_fragments;436 437    impl(struct llama_file * file, size_t prefetch, bool numa) {438        size = file->size();439        int fd = file->file_id();440        int flags = MAP_SHARED;441        if (numa) { prefetch = 0; }442#ifdef __linux__443        if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {444            LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",445                    strerror(errno));446        }447        if (prefetch) { flags |= MAP_POPULATE; }448#endif449        addr = mmap(NULL, file->size(), PROT_READ, flags, fd, 0);450        if (addr == MAP_FAILED) {451            throw std::runtime_error(format("mmap failed: %s", strerror(errno)));452        }453 454        if (prefetch > 0) {455            if (posix_madvise(addr, std::min(file->size(), prefetch), POSIX_MADV_WILLNEED)) {456                LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",457                        strerror(errno));458            }459        }460        if (numa) {461            if (posix_madvise(addr, file->size(), POSIX_MADV_RANDOM)) {462                LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",463                        strerror(errno));464            }465        }466 467        mapped_fragments.emplace_back(0, file->size());468    }469 470    static void align_range(size_t * first, size_t * last, size_t page_size) {471        size_t offset_in_page = *first & (page_size - 1);472        size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;473        *first += offset_to_page;474 475        *last = *last & ~(page_size - 1);476 477        if (*last <= *first) {478            *last = *first;479        }480    }481 482    void unmap_fragment(size_t first, size_t last) {483        int page_size = sysconf(_SC_PAGESIZE);484        align_range(&first, &last, page_size);485        size_t len = last - first;486 487        if (len == 0) {488            return;489        }490 491        GGML_ASSERT(first % page_size == 0);492        GGML_ASSERT(last % page_size == 0);493        GGML_ASSERT(last > first);494 495        void * next_page_start = (uint8_t *) addr + first;496 497        if (munmap(next_page_start, len)) {498            LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));499        }500 501        std::vector<std::pair<size_t, size_t>> new_mapped_fragments;502        for (const auto & frag : mapped_fragments) {503            if (frag.first < first && frag.second > last) {504                new_mapped_fragments.emplace_back(frag.first, first);505                new_mapped_fragments.emplace_back(last, frag.second);506            } else if (frag.first < first && frag.second > first) {507                new_mapped_fragments.emplace_back(frag.first, first);508            } else if (frag.first < last && frag.second > last) {509                new_mapped_fragments.emplace_back(last, frag.second);510            } else if (frag.first >= first && frag.second <= last) {511            } else {512                new_mapped_fragments.push_back(frag);513            }514        }515        mapped_fragments = std::move(new_mapped_fragments);516    }517 518    ~impl() {519        for (const auto & frag : mapped_fragments) {520            if (munmap((char *) addr + frag.first, frag.second - frag.first)) {521                LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));522            }523        }524    }525#elif defined(_WIN32)526    HANDLE hMapping = nullptr;527 528    impl(struct llama_file * file, size_t prefetch, bool numa) {529        GGML_UNUSED(numa);530 531        size = file->size();532 533        HANDLE hFile = (HANDLE) _get_osfhandle(file->file_id());534 535        hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);536 537        if (hMapping == NULL) {538            DWORD error = GetLastError();539            throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));540        }541 542        addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);543        DWORD error = GetLastError();544 545        if (addr == NULL) {546            CloseHandle(hMapping);547            throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));548        }549 550        if (prefetch > 0) {551#if _WIN32_WINNT >= 0x602552            BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);553            HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");554 555            pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");556 557            if (pPrefetchVirtualMemory) {558                WIN32_MEMORY_RANGE_ENTRY range;559                range.VirtualAddress = addr;560                range.NumberOfBytes = (SIZE_T) std::min(size, prefetch);561                if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {562                    LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",563                            llama_format_win_err(GetLastError()).c_str());564                }565            }566#else567            LLAMA_LOG_DEBUG("skipping PrefetchVirtualMemory because _WIN32_WINNT < 0x602\n");568#endif569        }570    }571 572    void unmap_fragment(size_t first, size_t last) {573        GGML_UNUSED(first);574        GGML_UNUSED(last);575    }576 577    ~impl() {578        if (hMapping) {579            if (addr) {580                if (!UnmapViewOfFile(addr)) {581                    LLAMA_LOG_WARN("warning: UnmapViewOfFile failed: %s\n",582                            llama_format_win_err(GetLastError()).c_str());583                }584            }585            if (!CloseHandle(hMapping)) {586                LLAMA_LOG_WARN("warning: CloseHandle failed: %s\n",587                        llama_format_win_err(GetLastError()).c_str());588            }589        }590    }591#else592    impl(struct llama_file * file, size_t prefetch, bool numa) {593        GGML_UNUSED(file);594        GGML_UNUSED(prefetch);595        GGML_UNUSED(numa);596 597        throw std::runtime_error("mmap not supported");598    }599 600    void unmap_fragment(size_t first, size_t last) {601        GGML_UNUSED(first);602        GGML_UNUSED(last);603 604        throw std::runtime_error("mmap not supported");605    }606#endif607 608    void * addr;609    size_t size;610};611 612llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique<impl>(file, prefetch, numa)) {}613llama_mmap::~llama_mmap() = default;614 615size_t llama_mmap::size() const { return pimpl->size; }616void * llama_mmap::addr() const { return pimpl->addr; }617 618void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); }619 620#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)621const bool llama_mmap::SUPPORTED  = true;622#else623const bool llama_mmap::SUPPORTED  = false;624#endif625 626// llama_mlock627 628struct llama_mlock::impl {629#ifdef _POSIX_MEMLOCK_RANGE630    static size_t lock_granularity() {631        return (size_t) sysconf(_SC_PAGESIZE);632    }633 634    bool raw_lock(const void * addr, size_t size) const {635        if (!mlock(addr, size)) {636            return true;637        }638 639#ifdef __APPLE__640#define MLOCK_SUGGESTION \641        "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \642        "decreasing 'vm.global_no_user_wire_amount'.  Also try increasing RLIMIT_MEMLOCK (ulimit -l).\n"643#else644#define MLOCK_SUGGESTION \645        "Try increasing RLIMIT_MEMLOCK ('ulimit -l' as root).\n"646#endif647 648        char* errmsg = std::strerror(errno);649        bool suggest = (errno == ENOMEM);650#if defined(TARGET_OS_VISION) || defined(TARGET_OS_TV) || defined(_AIX) || defined(__HAIKU__)651        // visionOS/tvOS/Haiku don't support RLIMIT_MEMLOCK652        // Skip resource limit checks on these platforms653        suggest = false;654#else655        struct rlimit lock_limit;656        if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) {657            suggest = false;658        }659        if (suggest && ((uint64_t)lock_limit.rlim_max > (uint64_t)lock_limit.rlim_cur + size)) {660            suggest = false;661        }662#endif663 664        LLAMA_LOG_WARN("warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",665                size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");666        return false;667    }668 669    static void raw_unlock(void * addr, size_t size) {670        if (munlock(addr, size)) {671            LLAMA_LOG_WARN("warning: failed to munlock buffer: %s\n", std::strerror(errno));672        }673    }674#elif defined(_WIN32)675    static size_t lock_granularity() {676        SYSTEM_INFO si;677        GetSystemInfo(&si);678        return (size_t) si.dwPageSize;679    }680 681    bool raw_lock(void * ptr, size_t len) const {682        for (int tries = 1; ; tries++) {683            if (VirtualLock(ptr, len)) {684                return true;685            }686            if (tries == 2) {687                LLAMA_LOG_WARN("warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",688                    len, size, llama_format_win_err(GetLastError()).c_str());689                return false;690            }691 692            SIZE_T min_ws_size, max_ws_size;693            if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {694                LLAMA_LOG_WARN("warning: GetProcessWorkingSetSize failed: %s\n",695                        llama_format_win_err(GetLastError()).c_str());696                return false;697            }698            size_t increment = len + 1048576;699            min_ws_size += increment;700            max_ws_size += increment;701            if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {702                LLAMA_LOG_WARN("warning: SetProcessWorkingSetSize failed: %s\n",703                        llama_format_win_err(GetLastError()).c_str());704                return false;705            }706        }707    }708 709    static void raw_unlock(void * ptr, size_t len) {710        if (!VirtualUnlock(ptr, len)) {711            LLAMA_LOG_WARN("warning: failed to VirtualUnlock buffer: %s\n",712                    llama_format_win_err(GetLastError()).c_str());713        }714    }715#else716    static size_t lock_granularity() {717        return (size_t) 65536;718    }719 720    bool raw_lock(const void * addr, size_t len) const {721        LLAMA_LOG_WARN("warning: mlock not supported on this system\n");722        return false;723    }724 725    static void raw_unlock(const void * addr, size_t len) {}726#endif727 728    impl() : addr(NULL), size(0), failed_already(false) {}729 730    void init(void * ptr) {731        GGML_ASSERT(addr == NULL && size == 0);732        addr = ptr;733    }734 735    void grow_to(size_t target_size) {736        GGML_ASSERT(addr);737        if (failed_already) {738            return;739        }740        size_t granularity = lock_granularity();741        target_size = (target_size + granularity - 1) & ~(granularity - 1);742        if (target_size > size) {743            if (raw_lock((uint8_t *) addr + size, target_size - size)) {744                size = target_size;745            } else {746                failed_already = true;747            }748        }749    }750 751    void * addr;752    size_t size;753 754    bool failed_already;755};756 757llama_mlock::llama_mlock() : pimpl(std::make_unique<impl>()) {}758llama_mlock::~llama_mlock() = default;759 760void llama_mlock::init(void * ptr) { pimpl->init(ptr); }761void llama_mlock::grow_to(size_t target_size) { pimpl->grow_to(target_size); }762 763#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)764const bool llama_mlock::SUPPORTED = true;765#else766const bool llama_mlock::SUPPORTED = false;767#endif768 769size_t llama_path_max() {770    return PATH_MAX;771}772