KBaba7/llama.cpp
0
1#include "llama-mmap.h"2 3#include "llama-impl.h"4 5#include "ggml.h"6 7#include <cstring>8#include <climits>9#include <stdexcept>10#include <cerrno>11 12#ifdef __has_include13 #if __has_include(<unistd.h>)14 #include <unistd.h>15 #if defined(_POSIX_MAPPED_FILES)16 #include <sys/mman.h>17 #include <fcntl.h>18 #endif19 #if defined(_POSIX_MEMLOCK_RANGE)20 #include <sys/resource.h>21 #endif22 #endif23#endif24 25#if defined(_WIN32)26 #define WIN32_LEAN_AND_MEAN27 #ifndef NOMINMAX28 #define NOMINMAX29 #endif30 #include <windows.h>31 #ifndef PATH_MAX32 #define PATH_MAX MAX_PATH33 #endif34 #include <io.h>35#endif36 37// TODO: consider moving to llama-impl.h if needed in more places38#if defined(_WIN32)39static std::string llama_format_win_err(DWORD err) {40 LPSTR buf;41 size_t size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,42 NULL, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&buf, 0, NULL);43 if (!size) {44 return "FormatMessageA failed";45 }46 std::string ret(buf, size);47 LocalFree(buf);48 return ret;49}50#endif51 52// llama_file53 54struct llama_file::impl {55#if defined(_WIN32)56 HANDLE fp_win32;57 std::string GetErrorMessageWin32(DWORD error_code) const {58 std::string ret;59 LPSTR lpMsgBuf = NULL;60 DWORD bufLen = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,61 NULL, error_code, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), (LPSTR)&lpMsgBuf, 0, NULL);62 if (!bufLen) {63 ret = format("Win32 error code: %lx", error_code);64 } else {65 ret = lpMsgBuf;66 LocalFree(lpMsgBuf);67 }68 69 return ret;70 }71 72 impl(const char * fname, const char * mode) {73 fp = ggml_fopen(fname, mode);74 if (fp == NULL) {75 throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));76 }77 fp_win32 = (HANDLE) _get_osfhandle(_fileno(fp));78 seek(0, SEEK_END);79 size = tell();80 seek(0, SEEK_SET);81 }82 83 size_t tell() const {84 LARGE_INTEGER li;85 li.QuadPart = 0;86 BOOL ret = SetFilePointerEx(fp_win32, li, &li, FILE_CURRENT);87 if (!ret) {88 throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));89 }90 91 return li.QuadPart;92 }93 94 void seek(size_t offset, int whence) const {95 static_assert(SEEK_SET == FILE_BEGIN, "SEEK_SET != FILE_BEGIN");96 static_assert(SEEK_CUR == FILE_CURRENT, "SEEK_CUR != FILE_CURRENT");97 static_assert(SEEK_END == FILE_END, "SEEK_END != FILE_END");98 99 LARGE_INTEGER li;100 li.QuadPart = offset;101 BOOL ret = SetFilePointerEx(fp_win32, li, NULL, whence);102 if (!ret) {103 throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));104 }105 }106 107 void read_raw(void * ptr, size_t len) const {108 size_t bytes_read = 0;109 while (bytes_read < len) {110 size_t chunk_size = std::min<size_t>(len - bytes_read, 64*1024*1024);111 DWORD chunk_read = 0;112 BOOL result = ReadFile(fp_win32, reinterpret_cast<char*>(ptr) + bytes_read, chunk_size, &chunk_read, NULL);113 if (!result) {114 throw std::runtime_error(format("read error: %s", GetErrorMessageWin32(GetLastError()).c_str()));115 }116 if (chunk_read < chunk_size || chunk_read == 0) {117 throw std::runtime_error("unexpectedly reached end of file");118 }119 120 bytes_read += chunk_read;121 }122 }123 124 uint32_t read_u32() const {125 uint32_t val;126 read_raw(&val, sizeof(val));127 return val;128 }129 130 void write_raw(const void * ptr, size_t len) const {131 size_t bytes_written = 0;132 while (bytes_written < len) {133 size_t chunk_size = std::min<size_t>(len - bytes_written, 64*1024*1024);134 DWORD chunk_written = 0;135 BOOL result = WriteFile(fp_win32, reinterpret_cast<char const*>(ptr) + bytes_written, chunk_size, &chunk_written, NULL);136 if (!result) {137 throw std::runtime_error(format("write error: %s", GetErrorMessageWin32(GetLastError()).c_str()));138 }139 if (chunk_written < chunk_size || chunk_written == 0) {140 throw std::runtime_error("unexpectedly failed to write bytes");141 }142 143 bytes_written += chunk_written;144 }145 }146 147 void write_u32(uint32_t val) const {148 write_raw(&val, sizeof(val));149 }150 151 ~impl() {152 if (fp) {153 std::fclose(fp);154 }155 }156#else157 impl(const char * fname, const char * mode) {158 fp = ggml_fopen(fname, mode);159 if (fp == NULL) {160 throw std::runtime_error(format("failed to open %s: %s", fname, strerror(errno)));161 }162 seek(0, SEEK_END);163 size = tell();164 seek(0, SEEK_SET);165 }166 167 size_t tell() const {168// TODO: this ifdef is never true?169#ifdef _WIN32170 __int64 ret = _ftelli64(fp);171#else172 long ret = std::ftell(fp);173#endif174 if (ret == -1) {175 throw std::runtime_error(format("ftell error: %s", strerror(errno)));176 }177 178 return (size_t) ret;179 }180 181 void seek(size_t offset, int whence) const {182// TODO: this ifdef is never true?183#ifdef _WIN32184 int ret = _fseeki64(fp, (__int64) offset, whence);185#else186 int ret = std::fseek(fp, (long) offset, whence);187#endif188 if (ret != 0) {189 throw std::runtime_error(format("seek error: %s", strerror(errno)));190 }191 }192 193 void read_raw(void * ptr, size_t len) const {194 if (len == 0) {195 return;196 }197 errno = 0;198 std::size_t ret = std::fread(ptr, len, 1, fp);199 if (ferror(fp)) {200 throw std::runtime_error(format("read error: %s", strerror(errno)));201 }202 if (ret != 1) {203 throw std::runtime_error("unexpectedly reached end of file");204 }205 }206 207 uint32_t read_u32() const {208 uint32_t ret;209 read_raw(&ret, sizeof(ret));210 return ret;211 }212 213 void write_raw(const void * ptr, size_t len) const {214 if (len == 0) {215 return;216 }217 errno = 0;218 size_t ret = std::fwrite(ptr, len, 1, fp);219 if (ret != 1) {220 throw std::runtime_error(format("write error: %s", strerror(errno)));221 }222 }223 224 void write_u32(uint32_t val) const {225 write_raw(&val, sizeof(val));226 }227 228 ~impl() {229 if (fp) {230 std::fclose(fp);231 }232 }233#endif234 235 FILE * fp;236 size_t size;237};238 239llama_file::llama_file(const char * fname, const char * mode) : pimpl(std::make_unique<impl>(fname, mode)) {}240llama_file::~llama_file() = default;241 242size_t llama_file::tell() const { return pimpl->tell(); }243size_t llama_file::size() const { return pimpl->size; }244 245int llama_file::file_id() const {246#ifdef _WIN32247 return _fileno(pimpl->fp);248#else249#if defined(fileno)250 return fileno(pimpl->fp);251#else252 return ::fileno(pimpl->fp);253#endif254#endif255}256 257void llama_file::seek(size_t offset, int whence) const { pimpl->seek(offset, whence); }258void llama_file::read_raw(void * ptr, size_t len) const { pimpl->read_raw(ptr, len); }259 260uint32_t llama_file::read_u32() const { return pimpl->read_u32(); }261 262void llama_file::write_raw(const void * ptr, size_t len) const { pimpl->write_raw(ptr, len); }263void llama_file::write_u32(uint32_t val) const { pimpl->write_u32(val); }264 265// llama_mmap266 267struct llama_mmap::impl {268#ifdef _POSIX_MAPPED_FILES269 std::vector<std::pair<size_t, size_t>> mapped_fragments;270 271 impl(struct llama_file * file, size_t prefetch, bool numa) {272 size = file->size();273 int fd = file->file_id();274 int flags = MAP_SHARED;275 if (numa) { prefetch = 0; }276#ifdef __linux__277 if (posix_fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL)) {278 LLAMA_LOG_WARN("warning: posix_fadvise(.., POSIX_FADV_SEQUENTIAL) failed: %s\n",279 strerror(errno));280 }281 if (prefetch) { flags |= MAP_POPULATE; }282#endif283 addr = mmap(NULL, file->size(), PROT_READ, flags, fd, 0);284 if (addr == MAP_FAILED) {285 throw std::runtime_error(format("mmap failed: %s", strerror(errno)));286 }287 288 if (prefetch > 0) {289 if (posix_madvise(addr, std::min(file->size(), prefetch), POSIX_MADV_WILLNEED)) {290 LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_WILLNEED) failed: %s\n",291 strerror(errno));292 }293 }294 if (numa) {295 if (posix_madvise(addr, file->size(), POSIX_MADV_RANDOM)) {296 LLAMA_LOG_WARN("warning: posix_madvise(.., POSIX_MADV_RANDOM) failed: %s\n",297 strerror(errno));298 }299 }300 301 mapped_fragments.emplace_back(0, file->size());302 }303 304 static void align_range(size_t * first, size_t * last, size_t page_size) {305 size_t offset_in_page = *first & (page_size - 1);306 size_t offset_to_page = offset_in_page == 0 ? 0 : page_size - offset_in_page;307 *first += offset_to_page;308 309 *last = *last & ~(page_size - 1);310 311 if (*last <= *first) {312 *last = *first;313 }314 }315 316 void unmap_fragment(size_t first, size_t last) {317 int page_size = sysconf(_SC_PAGESIZE);318 align_range(&first, &last, page_size);319 size_t len = last - first;320 321 if (len == 0) {322 return;323 }324 325 GGML_ASSERT(first % page_size == 0);326 GGML_ASSERT(last % page_size == 0);327 GGML_ASSERT(last > first);328 329 void * next_page_start = (uint8_t *) addr + first;330 331 if (munmap(next_page_start, len)) {332 LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));333 }334 335 std::vector<std::pair<size_t, size_t>> new_mapped_fragments;336 for (const auto & frag : mapped_fragments) {337 if (frag.first < first && frag.second > last) {338 new_mapped_fragments.emplace_back(frag.first, first);339 new_mapped_fragments.emplace_back(last, frag.second);340 } else if (frag.first < first && frag.second > first) {341 new_mapped_fragments.emplace_back(frag.first, first);342 } else if (frag.first < last && frag.second > last) {343 new_mapped_fragments.emplace_back(last, frag.second);344 } else if (frag.first >= first && frag.second <= last) {345 } else {346 new_mapped_fragments.push_back(frag);347 }348 }349 mapped_fragments = std::move(new_mapped_fragments);350 }351 352 ~impl() {353 for (const auto & frag : mapped_fragments) {354 if (munmap((char *) addr + frag.first, frag.second - frag.first)) {355 LLAMA_LOG_WARN("warning: munmap failed: %s\n", strerror(errno));356 }357 }358 }359#elif defined(_WIN32)360 impl(struct llama_file * file, size_t prefetch, bool numa) {361 GGML_UNUSED(numa);362 363 size = file->size();364 365 HANDLE hFile = (HANDLE) _get_osfhandle(file->file_id());366 367 HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, 0, NULL);368 369 if (hMapping == NULL) {370 DWORD error = GetLastError();371 throw std::runtime_error(format("CreateFileMappingA failed: %s", llama_format_win_err(error).c_str()));372 }373 374 addr = MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, 0);375 DWORD error = GetLastError();376 CloseHandle(hMapping);377 378 if (addr == NULL) {379 throw std::runtime_error(format("MapViewOfFile failed: %s", llama_format_win_err(error).c_str()));380 }381 382 if (prefetch > 0) {383#if _WIN32_WINNT >= 0x602384 BOOL (WINAPI *pPrefetchVirtualMemory) (HANDLE, ULONG_PTR, PWIN32_MEMORY_RANGE_ENTRY, ULONG);385 HMODULE hKernel32 = GetModuleHandleW(L"kernel32.dll");386 387 pPrefetchVirtualMemory = (decltype(pPrefetchVirtualMemory))(void *) GetProcAddress(hKernel32, "PrefetchVirtualMemory");388 389 if (pPrefetchVirtualMemory) {390 WIN32_MEMORY_RANGE_ENTRY range;391 range.VirtualAddress = addr;392 range.NumberOfBytes = (SIZE_T) std::min(size, prefetch);393 if (!pPrefetchVirtualMemory(GetCurrentProcess(), 1, &range, 0)) {394 LLAMA_LOG_WARN("warning: PrefetchVirtualMemory failed: %s\n",395 llama_format_win_err(GetLastError()).c_str());396 }397 }398#else399 throw std::runtime_error("PrefetchVirtualMemory unavailable");400#endif401 }402 }403 404 void unmap_fragment(size_t first, size_t last) {405 GGML_UNUSED(first);406 GGML_UNUSED(last);407 }408 409 ~impl() {410 if (!UnmapViewOfFile(addr)) {411 LLAMA_LOG_WARN("warning: UnmapViewOfFile failed: %s\n",412 llama_format_win_err(GetLastError()).c_str());413 }414 }415#else416 impl(struct llama_file * file, size_t prefetch, bool numa) {417 GGML_UNUSED(file);418 GGML_UNUSED(prefetch);419 GGML_UNUSED(numa);420 421 throw std::runtime_error("mmap not supported");422 }423 424 void unmap_fragment(size_t first, size_t last) {425 GGML_UNUSED(first);426 GGML_UNUSED(last);427 428 throw std::runtime_error("mmap not supported");429 }430#endif431 432 void * addr;433 size_t size;434};435 436llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa) : pimpl(std::make_unique<impl>(file, prefetch, numa)) {}437llama_mmap::~llama_mmap() = default;438 439size_t llama_mmap::size() const { return pimpl->size; }440void * llama_mmap::addr() const { return pimpl->addr; }441 442void llama_mmap::unmap_fragment(size_t first, size_t last) { pimpl->unmap_fragment(first, last); }443 444#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)445const bool llama_mmap::SUPPORTED = true;446#else447const bool llama_mmap::SUPPORTED = false;448#endif449 450// llama_mlock451 452struct llama_mlock::impl {453#ifdef _POSIX_MEMLOCK_RANGE454 static size_t lock_granularity() {455 return (size_t) sysconf(_SC_PAGESIZE);456 }457 458 bool raw_lock(const void * addr, size_t size) const {459 if (!mlock(addr, size)) {460 return true;461 }462 463#ifdef __APPLE__464#define MLOCK_SUGGESTION \465 "Try increasing the sysctl values 'vm.user_wire_limit' and 'vm.global_user_wire_limit' and/or " \466 "decreasing 'vm.global_no_user_wire_amount'. Also try increasing RLIMIT_MEMLOCK (ulimit -l).\n"467#else468#define MLOCK_SUGGESTION \469 "Try increasing RLIMIT_MEMLOCK ('ulimit -l' as root).\n"470#endif471 472 char* errmsg = std::strerror(errno);473 bool suggest = (errno == ENOMEM);474 475 struct rlimit lock_limit;476 if (suggest && getrlimit(RLIMIT_MEMLOCK, &lock_limit)) {477 suggest = false;478 }479 if (suggest && (lock_limit.rlim_max > lock_limit.rlim_cur + size)) {480 suggest = false;481 }482 483 LLAMA_LOG_WARN("warning: failed to mlock %zu-byte buffer (after previously locking %zu bytes): %s\n%s",484 size, this->size, errmsg, suggest ? MLOCK_SUGGESTION : "");485 return false;486 }487 488 static void raw_unlock(void * addr, size_t size) {489 if (munlock(addr, size)) {490 LLAMA_LOG_WARN("warning: failed to munlock buffer: %s\n", std::strerror(errno));491 }492 }493#elif defined(_WIN32)494 static size_t lock_granularity() {495 SYSTEM_INFO si;496 GetSystemInfo(&si);497 return (size_t) si.dwPageSize;498 }499 500 bool raw_lock(void * ptr, size_t len) const {501 for (int tries = 1; ; tries++) {502 if (VirtualLock(ptr, len)) {503 return true;504 }505 if (tries == 2) {506 LLAMA_LOG_WARN("warning: failed to VirtualLock %zu-byte buffer (after previously locking %zu bytes): %s\n",507 len, size, llama_format_win_err(GetLastError()).c_str());508 return false;509 }510 511 SIZE_T min_ws_size, max_ws_size;512 if (!GetProcessWorkingSetSize(GetCurrentProcess(), &min_ws_size, &max_ws_size)) {513 LLAMA_LOG_WARN("warning: GetProcessWorkingSetSize failed: %s\n",514 llama_format_win_err(GetLastError()).c_str());515 return false;516 }517 size_t increment = len + 1048576;518 min_ws_size += increment;519 max_ws_size += increment;520 if (!SetProcessWorkingSetSize(GetCurrentProcess(), min_ws_size, max_ws_size)) {521 LLAMA_LOG_WARN("warning: SetProcessWorkingSetSize failed: %s\n",522 llama_format_win_err(GetLastError()).c_str());523 return false;524 }525 }526 }527 528 static void raw_unlock(void * ptr, size_t len) {529 if (!VirtualUnlock(ptr, len)) {530 LLAMA_LOG_WARN("warning: failed to VirtualUnlock buffer: %s\n",531 llama_format_win_err(GetLastError()).c_str());532 }533 }534#else535 static size_t lock_granularity() {536 return (size_t) 65536;537 }538 539 bool raw_lock(const void * addr, size_t len) const {540 LLAMA_LOG_WARN("warning: mlock not supported on this system\n");541 return false;542 }543 544 static void raw_unlock(const void * addr, size_t len) {}545#endif546 547 impl() : addr(NULL), size(0), failed_already(false) {}548 549 void init(void * ptr) {550 GGML_ASSERT(addr == NULL && size == 0);551 addr = ptr;552 }553 554 void grow_to(size_t target_size) {555 GGML_ASSERT(addr);556 if (failed_already) {557 return;558 }559 size_t granularity = lock_granularity();560 target_size = (target_size + granularity - 1) & ~(granularity - 1);561 if (target_size > size) {562 if (raw_lock((uint8_t *) addr + size, target_size - size)) {563 size = target_size;564 } else {565 failed_already = true;566 }567 }568 }569 570 void * addr;571 size_t size;572 573 bool failed_already;574};575 576llama_mlock::llama_mlock() : pimpl(std::make_unique<impl>()) {}577llama_mlock::~llama_mlock() = default;578 579void llama_mlock::init(void * ptr) { pimpl->init(ptr); }580void llama_mlock::grow_to(size_t target_size) { pimpl->grow_to(target_size); }581 582#if defined(_POSIX_MEMLOCK_RANGE) || defined(_WIN32)583const bool llama_mlock::SUPPORTED = true;584#else585const bool llama_mlock::SUPPORTED = false;586#endif587 588size_t llama_path_max() {589 return PATH_MAX;590}591