KBaba7/llama.cpp
0
1#include "ggml-backend-impl.h"2#include "ggml-backend.h"3#include "ggml-impl.h"4#include <algorithm>5#include <codecvt>6#include <cstring>7#include <filesystem>8#include <locale>9#include <memory>10#include <string>11#include <type_traits>12#include <vector>13 14#ifdef _WIN3215# define WIN32_LEAN_AND_MEAN16# ifndef NOMINMAX17# define NOMINMAX18# endif19# include <windows.h>20#elif defined(__APPLE__)21# include <mach-o/dyld.h>22# include <dlfcn.h>23#else24# include <dlfcn.h>25# include <unistd.h>26#endif27 28// Backend registry29#ifdef GGML_USE_CPU30#include "ggml-cpu.h"31#endif32 33#ifdef GGML_USE_CUDA34#include "ggml-cuda.h"35#endif36 37#ifdef GGML_USE_METAL38#include "ggml-metal.h"39#endif40 41#ifdef GGML_USE_SYCL42#include "ggml-sycl.h"43#endif44 45#ifdef GGML_USE_VULKAN46#include "ggml-vulkan.h"47#endif48 49#ifdef GGML_USE_OPENCL50#include "ggml-opencl.h"51#endif52 53#ifdef GGML_USE_BLAS54#include "ggml-blas.h"55#endif56 57#ifdef GGML_USE_RPC58#include "ggml-rpc.h"59#endif60 61#ifdef GGML_USE_CANN62#include "ggml-cann.h"63#endif64 65#ifdef GGML_USE_KOMPUTE66#include "ggml-kompute.h"67#endif68 69// disable C++17 deprecation warning for std::codecvt_utf870#if defined(__clang__)71# pragma clang diagnostic push72# pragma clang diagnostic ignored "-Wdeprecated-declarations"73#endif74 75static std::wstring utf8_to_utf16(const std::string & str) {76 std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;77 return converter.from_bytes(str);78}79 80static std::string utf16_to_utf8(const std::wstring & str) {81 std::wstring_convert<std::codecvt_utf8_utf16<wchar_t>> converter;82 return converter.to_bytes(str);83}84 85#if defined(__clang__)86# pragma clang diagnostic pop87#endif88 89#ifdef _WIN3290 91using dl_handle = std::remove_pointer_t<HMODULE>;92 93struct dl_handle_deleter {94 void operator()(HMODULE handle) {95 FreeLibrary(handle);96 }97};98 99static dl_handle * dl_load_library(const std::wstring & path) {100 // suppress error dialogs for missing DLLs101 DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);102 SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);103 104 HMODULE handle = LoadLibraryW(path.c_str());105 106 SetErrorMode(old_mode);107 108 return handle;109}110 111static void * dl_get_sym(dl_handle * handle, const char * name) {112 DWORD old_mode = SetErrorMode(SEM_FAILCRITICALERRORS);113 SetErrorMode(old_mode | SEM_FAILCRITICALERRORS);114 115 void * p = (void *) GetProcAddress(handle, name);116 117 SetErrorMode(old_mode);118 119 return p;120}121 122#else123 124using dl_handle = void;125 126struct dl_handle_deleter {127 void operator()(void * handle) {128 dlclose(handle);129 }130};131 132static void * dl_load_library(const std::wstring & path) {133 dl_handle * handle = dlopen(utf16_to_utf8(path).c_str(), RTLD_NOW | RTLD_LOCAL);134 135 return handle;136}137 138static void * dl_get_sym(dl_handle * handle, const char * name) {139 return dlsym(handle, name);140}141 142#endif143 144using dl_handle_ptr = std::unique_ptr<dl_handle, dl_handle_deleter>;145 146struct ggml_backend_reg_entry {147 ggml_backend_reg_t reg;148 dl_handle_ptr handle;149};150 151struct ggml_backend_registry {152 std::vector<ggml_backend_reg_entry> backends;153 std::vector<ggml_backend_dev_t> devices;154 155 ggml_backend_registry() {156#ifdef GGML_USE_CUDA157 register_backend(ggml_backend_cuda_reg());158#endif159#ifdef GGML_USE_METAL160 register_backend(ggml_backend_metal_reg());161#endif162#ifdef GGML_USE_SYCL163 register_backend(ggml_backend_sycl_reg());164#endif165#ifdef GGML_USE_VULKAN166 register_backend(ggml_backend_vk_reg());167#endif168#ifdef GGML_USE_OPENCL169 register_backend(ggml_backend_opencl_reg());170#endif171#ifdef GGML_USE_CANN172 register_backend(ggml_backend_cann_reg());173#endif174#ifdef GGML_USE_BLAS175 register_backend(ggml_backend_blas_reg());176#endif177#ifdef GGML_USE_RPC178 register_backend(ggml_backend_rpc_reg());179#endif180#ifdef GGML_USE_KOMPUTE181 register_backend(ggml_backend_kompute_reg());182#endif183#ifdef GGML_USE_CPU184 register_backend(ggml_backend_cpu_reg());185#endif186 }187 188 ~ggml_backend_registry() {189 // FIXME: backends cannot be safely unloaded without a function to destroy all the backend resources,190 // since backend threads may still be running and accessing resources from the dynamic library191 for (auto & entry : backends) {192 if (entry.handle) {193 entry.handle.release(); // NOLINT194 }195 }196 }197 198 void register_backend(ggml_backend_reg_t reg, dl_handle_ptr handle = nullptr) {199 if (!reg) {200 return;201 }202 203#ifndef NDEBUG204 GGML_LOG_DEBUG("%s: registered backend %s (%zu devices)\n",205 __func__, ggml_backend_reg_name(reg), ggml_backend_reg_dev_count(reg));206#endif207 backends.push_back({ reg, std::move(handle) });208 for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); i++) {209 register_device(ggml_backend_reg_dev_get(reg, i));210 }211 }212 213 void register_device(ggml_backend_dev_t device) {214#ifndef NDEBUG215 GGML_LOG_DEBUG("%s: registered device %s (%s)\n", __func__, ggml_backend_dev_name(device), ggml_backend_dev_description(device));216#endif217 devices.push_back(device);218 }219 220 ggml_backend_reg_t load_backend(const std::wstring & path, bool silent) {221 dl_handle_ptr handle { dl_load_library(path) };222 if (!handle) {223 if (!silent) {224 GGML_LOG_ERROR("%s: failed to load %s\n", __func__, utf16_to_utf8(path).c_str());225 }226 return nullptr;227 }228 229 auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score");230 if (score_fn && score_fn() == 0) {231 if (!silent) {232 GGML_LOG_INFO("%s: backend %s is not supported on this system\n", __func__, utf16_to_utf8(path).c_str());233 }234 return nullptr;235 }236 237 auto backend_init_fn = (ggml_backend_init_t) dl_get_sym(handle.get(), "ggml_backend_init");238 if (!backend_init_fn) {239 if (!silent) {240 GGML_LOG_ERROR("%s: failed to find ggml_backend_init in %s\n", __func__, utf16_to_utf8(path).c_str());241 }242 return nullptr;243 }244 245 ggml_backend_reg_t reg = backend_init_fn();246 if (!reg || reg->api_version != GGML_BACKEND_API_VERSION) {247 if (!silent) {248 if (!reg) {249 GGML_LOG_ERROR("%s: failed to initialize backend from %s: ggml_backend_init returned NULL\n", __func__, utf16_to_utf8(path).c_str());250 } else {251 GGML_LOG_ERROR("%s: failed to initialize backend from %s: incompatible API version (backend: %d, current: %d)\n",252 __func__, utf16_to_utf8(path).c_str(), reg->api_version, GGML_BACKEND_API_VERSION);253 }254 }255 return nullptr;256 }257 258 GGML_LOG_INFO("%s: loaded %s backend from %s\n", __func__, ggml_backend_reg_name(reg), utf16_to_utf8(path).c_str());259 260 register_backend(reg, std::move(handle));261 262 return reg;263 }264 265 void unload_backend(ggml_backend_reg_t reg, bool silent) {266 auto it = std::find_if(backends.begin(), backends.end(),267 [reg](const ggml_backend_reg_entry & entry) { return entry.reg == reg; });268 269 if (it == backends.end()) {270 if (!silent) {271 GGML_LOG_ERROR("%s: backend not found\n", __func__);272 }273 return;274 }275 276 if (!silent) {277 GGML_LOG_DEBUG("%s: unloading %s backend\n", __func__, ggml_backend_reg_name(reg));278 }279 280 // remove devices281 devices.erase(282 std::remove_if(devices.begin(), devices.end(),283 [reg](ggml_backend_dev_t dev) { return ggml_backend_dev_backend_reg(dev) == reg; }),284 devices.end());285 286 // remove backend287 backends.erase(it);288 }289};290 291static ggml_backend_registry & get_reg() {292 static ggml_backend_registry reg;293 return reg;294}295 296// Internal API297void ggml_backend_register(ggml_backend_reg_t reg) {298 get_reg().register_backend(reg);299}300 301void ggml_backend_device_register(ggml_backend_dev_t device) {302 get_reg().register_device(device);303}304 305// Backend (reg) enumeration306static bool striequals(const char * a, const char * b) {307 for (; *a && *b; a++, b++) {308 if (std::tolower(*a) != std::tolower(*b)) {309 return false;310 }311 }312 return *a == *b;313}314 315size_t ggml_backend_reg_count() {316 return get_reg().backends.size();317}318 319ggml_backend_reg_t ggml_backend_reg_get(size_t index) {320 GGML_ASSERT(index < ggml_backend_reg_count());321 return get_reg().backends[index].reg;322}323 324ggml_backend_reg_t ggml_backend_reg_by_name(const char * name) {325 for (size_t i = 0; i < ggml_backend_reg_count(); i++) {326 ggml_backend_reg_t reg = ggml_backend_reg_get(i);327 if (striequals(ggml_backend_reg_name(reg), name)) {328 return reg;329 }330 }331 return nullptr;332}333 334// Device enumeration335size_t ggml_backend_dev_count() {336 return get_reg().devices.size();337}338 339ggml_backend_dev_t ggml_backend_dev_get(size_t index) {340 GGML_ASSERT(index < ggml_backend_dev_count());341 return get_reg().devices[index];342}343 344ggml_backend_dev_t ggml_backend_dev_by_name(const char * name) {345 for (size_t i = 0; i < ggml_backend_dev_count(); i++) {346 ggml_backend_dev_t dev = ggml_backend_dev_get(i);347 if (striequals(ggml_backend_dev_name(dev), name)) {348 return dev;349 }350 }351 return nullptr;352}353 354ggml_backend_dev_t ggml_backend_dev_by_type(enum ggml_backend_dev_type type) {355 for (size_t i = 0; i < ggml_backend_dev_count(); i++) {356 ggml_backend_dev_t dev = ggml_backend_dev_get(i);357 if (ggml_backend_dev_type(dev) == type) {358 return dev;359 }360 }361 return nullptr;362}363 364// Convenience functions365ggml_backend_t ggml_backend_init_by_name(const char * name, const char * params) {366 ggml_backend_dev_t dev = ggml_backend_dev_by_name(name);367 if (!dev) {368 return nullptr;369 }370 return ggml_backend_dev_init(dev, params);371}372 373ggml_backend_t ggml_backend_init_by_type(enum ggml_backend_dev_type type, const char * params) {374 ggml_backend_dev_t dev = ggml_backend_dev_by_type(type);375 if (!dev) {376 return nullptr;377 }378 return ggml_backend_dev_init(dev, params);379}380 381ggml_backend_t ggml_backend_init_best(void) {382 ggml_backend_dev_t dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_GPU);383 if (!dev) {384 dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);385 }386 if (!dev) {387 return nullptr;388 }389 return ggml_backend_dev_init(dev, nullptr);390}391 392// Dynamic loading393ggml_backend_reg_t ggml_backend_load(const char * path) {394 return get_reg().load_backend(utf8_to_utf16(path), false);395}396 397void ggml_backend_unload(ggml_backend_reg_t reg) {398 get_reg().unload_backend(reg, true);399}400 401static std::wstring get_executable_path() {402#if defined(__APPLE__)403 // get executable path404 std::vector<char> path;405 uint32_t size;406 while (true) {407 size = path.size();408 if (_NSGetExecutablePath(path.data(), &size) == 0) {409 break;410 }411 path.resize(size);412 }413 std::string base_path(path.data(), size);414 // remove executable name415 auto last_slash = base_path.find_last_of('/');416 if (last_slash != std::string::npos) {417 base_path = base_path.substr(0, last_slash);418 }419 return utf8_to_utf16(base_path + "/");420#elif defined(__linux__) || defined(__FreeBSD__)421 std::string base_path = ".";422 std::vector<char> path(1024);423 while (true) {424 // get executable path425# if defined(__linux__)426 ssize_t len = readlink("/proc/self/exe", path.data(), path.size());427# elif defined(__FreeBSD__)428 ssize_t len = readlink("/proc/curproc/file", path.data(), path.size());429# endif430 if (len == -1) {431 break;432 }433 if (len < (ssize_t) path.size()) {434 base_path = std::string(path.data(), len);435 // remove executable name436 auto last_slash = base_path.find_last_of('/');437 if (last_slash != std::string::npos) {438 base_path = base_path.substr(0, last_slash);439 }440 break;441 }442 path.resize(path.size() * 2);443 }444 445 return utf8_to_utf16(base_path + "/");446#elif defined(_WIN32)447 std::vector<wchar_t> path(MAX_PATH);448 DWORD len = GetModuleFileNameW(NULL, path.data(), path.size());449 if (len == 0) {450 return {};451 }452 std::wstring base_path(path.data(), len);453 // remove executable name454 auto last_slash = base_path.find_last_of('\\');455 if (last_slash != std::string::npos) {456 base_path = base_path.substr(0, last_slash);457 }458 return base_path + L"\\";459#else460 return {};461#endif462}463 464static std::wstring backend_filename_prefix() {465#ifdef _WIN32466 return L"ggml-";467#else468 return L"libggml-";469#endif470}471 472static std::wstring backend_filename_suffix() {473#ifdef _WIN32474 return L".dll";475#else476 return L".so";477#endif478}479 480static std::wstring path_separator() {481#ifdef _WIN32482 return L"\\";483#else484 return L"/";485#endif486}487 488static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, const char * user_search_path) {489 // enumerate all the files that match [lib]ggml-name-*.[so|dll] in the search paths490 // TODO: search system paths491 std::wstring file_prefix = backend_filename_prefix() + utf8_to_utf16(name) + L"-";492 std::vector<std::wstring> search_paths;493 if (user_search_path == nullptr) {494 search_paths.push_back(L"." + path_separator());495 search_paths.push_back(get_executable_path());496 } else {497 search_paths.push_back(utf8_to_utf16(user_search_path) + path_separator());498 }499 500 int best_score = 0;501 std::wstring best_path;502 503 namespace fs = std::filesystem;504 for (const auto & search_path : search_paths) {505 if (!fs::exists(search_path)) {506 continue;507 }508 fs::directory_iterator dir_it(search_path, fs::directory_options::skip_permission_denied);509 for (const auto & entry : dir_it) {510 if (entry.is_regular_file()) {511 std::wstring filename = entry.path().filename().wstring();512 std::wstring ext = entry.path().extension().wstring();513 if (filename.find(file_prefix) == 0 && ext == backend_filename_suffix()) {514 dl_handle_ptr handle { dl_load_library(entry.path().wstring()) };515 if (!handle && !silent) {516 GGML_LOG_ERROR("%s: failed to load %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str());517 }518 if (handle) {519 auto score_fn = (ggml_backend_score_t) dl_get_sym(handle.get(), "ggml_backend_score");520 if (score_fn) {521 int s = score_fn();522#ifndef NDEBUG523 GGML_LOG_DEBUG("%s: %s score: %d\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str(), s);524#endif525 if (s > best_score) {526 best_score = s;527 best_path = entry.path().wstring();528 }529 } else {530 if (!silent) {531 GGML_LOG_INFO("%s: failed to find ggml_backend_score in %s\n", __func__, utf16_to_utf8(entry.path().wstring()).c_str());532 }533 }534 }535 }536 }537 }538 }539 540 if (best_score == 0) {541 // try to load the base backend542 for (const auto & search_path : search_paths) {543 std::wstring path = search_path + backend_filename_prefix() + utf8_to_utf16(name) + backend_filename_suffix();544 if (fs::exists(path)) {545 return get_reg().load_backend(path, silent);546 }547 }548 return nullptr;549 }550 551 return get_reg().load_backend(best_path, silent);552}553 554void ggml_backend_load_all() {555 ggml_backend_load_all_from_path(nullptr);556}557 558void ggml_backend_load_all_from_path(const char * dir_path) {559#ifdef NDEBUG560 bool silent = true;561#else562 bool silent = false;563#endif564 565 ggml_backend_load_best("blas", silent, dir_path);566 ggml_backend_load_best("cann", silent, dir_path);567 ggml_backend_load_best("cuda", silent, dir_path);568 ggml_backend_load_best("hip", silent, dir_path);569 ggml_backend_load_best("kompute", silent, dir_path);570 ggml_backend_load_best("metal", silent, dir_path);571 ggml_backend_load_best("rpc", silent, dir_path);572 ggml_backend_load_best("sycl", silent, dir_path);573 ggml_backend_load_best("vulkan", silent, dir_path);574 ggml_backend_load_best("opencl", silent, dir_path);575 ggml_backend_load_best("musa", silent, dir_path);576 ggml_backend_load_best("cpu", silent, dir_path);577 // check the environment variable GGML_BACKEND_PATH to load an out-of-tree backend578 const char * backend_path = std::getenv("GGML_BACKEND_PATH");579 if (backend_path) {580 ggml_backend_load(backend_path);581 }582}583