echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0604
1#include "llama-model-loader.h"2 3#include "ggml-alloc.h"4#include "ggml.h"5#include "gguf.h"6#include "llama-hparams.h"7 8#include <algorithm>9#include <array>10#include <cinttypes>11#include <cstdint>12#include <cstring>13#include <future>14#include <regex>15 16static const size_t kiB = 1024;17static const size_t MiB = 1024*kiB;18static const size_t GiB = 1024*MiB;19 20const char * llama_file_version_name(llama_fver version) {21 switch (version) {22 case GGUF_FILE_VERSION_V1: return "GGUF V1 (support until nov 2023)";23 case GGUF_FILE_VERSION_V2: return "GGUF V2";24 case GGUF_FILE_VERSION_V3: return "GGUF V3 (latest)";25 }26 27 return "unknown";28}29 30static std::string llama_model_ftype_name(llama_ftype ftype) {31 if (ftype & LLAMA_FTYPE_GUESSED) {32 return llama_model_ftype_name((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) + " (guessed)";33 }34 35 switch (ftype) {36 case LLAMA_FTYPE_ALL_F32: return "all F32";37 case LLAMA_FTYPE_MOSTLY_F16: return "F16";38 case LLAMA_FTYPE_MOSTLY_BF16: return "BF16";39 case LLAMA_FTYPE_MOSTLY_Q1_0: return "Q1_0";40 case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0";41 case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1";42 case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0";43 case LLAMA_FTYPE_MOSTLY_Q5_1: return "Q5_1";44 case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0";45 case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return "MXFP4 MoE";46 case LLAMA_FTYPE_MOSTLY_NVFP4: return "NVFP4";47 case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K - Medium";48 case LLAMA_FTYPE_MOSTLY_Q2_K_S: return "Q2_K - Small";49 case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small";50 case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium";51 case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large";52 case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "Q4_K - Small";53 case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "Q4_K - Medium";54 case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small";55 case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium";56 case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K";57 case LLAMA_FTYPE_MOSTLY_TQ1_0: return "TQ1_0 - 1.69 bpw ternary";58 case LLAMA_FTYPE_MOSTLY_TQ2_0: return "TQ2_0 - 2.06 bpw ternary";59 case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return "IQ2_XXS - 2.0625 bpw";60 case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw";61 case LLAMA_FTYPE_MOSTLY_IQ2_S: return "IQ2_S - 2.5 bpw";62 case LLAMA_FTYPE_MOSTLY_IQ2_M: return "IQ2_M - 2.7 bpw";63 case LLAMA_FTYPE_MOSTLY_IQ3_XS: return "IQ3_XS - 3.3 bpw";64 case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return "IQ3_XXS - 3.0625 bpw";65 case LLAMA_FTYPE_MOSTLY_IQ1_S: return "IQ1_S - 1.5625 bpw";66 case LLAMA_FTYPE_MOSTLY_IQ1_M: return "IQ1_M - 1.75 bpw";67 case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw";68 case LLAMA_FTYPE_MOSTLY_IQ4_XS: return "IQ4_XS - 4.25 bpw";69 case LLAMA_FTYPE_MOSTLY_IQ3_S: return "IQ3_S - 3.4375 bpw";70 case LLAMA_FTYPE_MOSTLY_IQ3_M: return "IQ3_S mix - 3.66 bpw";71 72 default: return "unknown, may not work";73 }74}75 76// return a list of splits for a given path77// for example, given "<name>-00002-of-00004.gguf", returns list of all 4 splits78static std::vector<std::string> llama_get_list_splits(const std::string & path, const int idx, const int n_split) {79 std::vector<std::string> paths;80 std::string split_prefix;81 std::vector<char> buf(llama_path_max(), 0);82 83 {84 int ret = llama_split_prefix(buf.data(), buf.size(), path.c_str(), idx, n_split);85 if (!ret) {86 throw std::runtime_error(format("invalid split file name: %s", path.c_str()));87 }88 split_prefix = std::string(buf.data(), ret);89 }90 91 if (split_prefix.empty()) {92 throw std::runtime_error(format("invalid split file: %s", path.c_str()));93 }94 95 for (int idx = 0; idx < n_split; ++idx) {96 int ret = llama_split_path(buf.data(), buf.size(), split_prefix.c_str(), idx, n_split);97 paths.push_back(std::string(buf.data(), ret));98 }99 100 return paths;101}102 103namespace GGUFMeta {104 template <typename T, gguf_type gt_, T (*gfun)(const gguf_context *, const int64_t)>105 struct GKV_Base_Type {106 static constexpr gguf_type gt = gt_;107 108 static T getter(const gguf_context * ctx, const int kid) {109 return gfun(ctx, kid);110 }111 };112 113 template<typename T> struct GKV_Base;114 115 template<> struct GKV_Base<bool >: GKV_Base_Type<bool, GGUF_TYPE_BOOL, gguf_get_val_bool> {};116 template<> struct GKV_Base<uint8_t >: GKV_Base_Type<uint8_t, GGUF_TYPE_UINT8, gguf_get_val_u8 > {};117 template<> struct GKV_Base<uint16_t >: GKV_Base_Type<uint16_t, GGUF_TYPE_UINT16, gguf_get_val_u16 > {};118 template<> struct GKV_Base<uint32_t >: GKV_Base_Type<uint32_t, GGUF_TYPE_UINT32, gguf_get_val_u32 > {};119 template<> struct GKV_Base<uint64_t >: GKV_Base_Type<uint64_t, GGUF_TYPE_UINT64, gguf_get_val_u64 > {};120 template<> struct GKV_Base<int8_t >: GKV_Base_Type<int8_t, GGUF_TYPE_INT8, gguf_get_val_i8 > {};121 template<> struct GKV_Base<int16_t >: GKV_Base_Type<int16_t, GGUF_TYPE_INT16, gguf_get_val_i16 > {};122 template<> struct GKV_Base<int32_t >: GKV_Base_Type<int32_t, GGUF_TYPE_INT32, gguf_get_val_i32 > {};123 template<> struct GKV_Base<int64_t >: GKV_Base_Type<int64_t, GGUF_TYPE_INT64, gguf_get_val_i64 > {};124 template<> struct GKV_Base<float >: GKV_Base_Type<float, GGUF_TYPE_FLOAT32, gguf_get_val_f32 > {};125 template<> struct GKV_Base<double >: GKV_Base_Type<double, GGUF_TYPE_FLOAT64, gguf_get_val_f64 > {};126 template<> struct GKV_Base<const char *>: GKV_Base_Type<const char *, GGUF_TYPE_STRING, gguf_get_val_str > {};127 128 template<> struct GKV_Base<std::string> {129 static constexpr gguf_type gt = GGUF_TYPE_STRING;130 131 static std::string getter(const gguf_context * ctx, const int kid) {132 return gguf_get_val_str(ctx, kid);133 }134 };135 136 struct ArrayInfo {137 const gguf_type gt;138 const size_t length;139 const void * data;140 };141 142 template<> struct GKV_Base<ArrayInfo> {143 public:144 static constexpr gguf_type gt = GGUF_TYPE_ARRAY;145 static ArrayInfo getter(const gguf_context *ctx, const int k) {146 const enum gguf_type arr_type = gguf_get_arr_type(ctx, k);147 return ArrayInfo {148 arr_type,149 size_t(gguf_get_arr_n(ctx, k)),150 arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx, k),151 };152 }153 };154 155 template<typename T>156 class GKV : public GKV_Base<T> {157 GKV() = delete;158 159 public:160 static T get_kv(const gguf_context * ctx, const int k) {161 const enum gguf_type kt = gguf_get_kv_type(ctx, k);162 163 if (kt != GKV::gt) {164 throw std::runtime_error(format("key %s has wrong type %s but expected type %s",165 gguf_get_key(ctx, k), gguf_type_name(kt), gguf_type_name(GKV::gt)));166 }167 return GKV::getter(ctx, k);168 }169 170 static const char * override_type_to_str(const llama_model_kv_override_type ty) {171 switch (ty) {172 case LLAMA_KV_OVERRIDE_TYPE_BOOL: return "bool";173 case LLAMA_KV_OVERRIDE_TYPE_INT: return "int";174 case LLAMA_KV_OVERRIDE_TYPE_FLOAT: return "float";175 case LLAMA_KV_OVERRIDE_TYPE_STR: return "str";176 }177 return "unknown";178 }179 180 static bool validate_override(const llama_model_kv_override_type expected_type, const struct llama_model_kv_override * ovrd) {181 if (!ovrd) { return false; }182 if (ovrd->tag == expected_type) {183 LLAMA_LOG_INFO("%s: Using metadata override (%5s) '%s' = ",184 __func__, override_type_to_str(ovrd->tag), ovrd->key);185 switch (ovrd->tag) {186 case LLAMA_KV_OVERRIDE_TYPE_BOOL: {187 LLAMA_LOG_INFO("%s\n", ovrd->val_bool ? "true" : "false");188 } break;189 case LLAMA_KV_OVERRIDE_TYPE_INT: {190 LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->val_i64);191 } break;192 case LLAMA_KV_OVERRIDE_TYPE_FLOAT: {193 LLAMA_LOG_INFO("%.6f\n", ovrd->val_f64);194 } break;195 case LLAMA_KV_OVERRIDE_TYPE_STR: {196 LLAMA_LOG_INFO("%s\n", ovrd->val_str);197 } break;198 default:199 // Shouldn't be possible to end up here, but just in case...200 throw std::runtime_error(201 format("Unsupported attempt to override %s type for metadata key %s\n",202 override_type_to_str(ovrd->tag), ovrd->key));203 }204 return true;205 }206 LLAMA_LOG_WARN("%s: Warning: Bad metadata override type for key '%s', expected %s but got %s\n",207 __func__, ovrd->key, override_type_to_str(expected_type), override_type_to_str(ovrd->tag));208 return false;209 }210 211 template<typename OT>212 static typename std::enable_if<std::is_same<OT, bool>::value, bool>::type213 try_override(OT & target, const struct llama_model_kv_override * ovrd) {214 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, ovrd)) {215 target = ovrd->val_bool;216 return true;217 }218 return false;219 }220 221 template<typename OT>222 static typename std::enable_if<!std::is_same<OT, bool>::value && std::is_integral<OT>::value, bool>::type223 try_override(OT & target, const struct llama_model_kv_override * ovrd) {224 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, ovrd)) {225 target = ovrd->val_i64;226 return true;227 }228 return false;229 }230 231 template<typename OT>232 static typename std::enable_if<std::is_floating_point<OT>::value, bool>::type233 try_override(T & target, const struct llama_model_kv_override * ovrd) {234 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, ovrd)) {235 target = ovrd->val_f64;236 return true;237 }238 return false;239 }240 241 template<typename OT>242 static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::type243 try_override(T & target, const struct llama_model_kv_override * ovrd) {244 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_STR, ovrd)) {245 target = ovrd->val_str;246 return true;247 }248 return false;249 }250 251 static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override * ovrd = nullptr) {252 if (try_override<T>(target, ovrd)) {253 return true;254 }255 if (k < 0) { return false; }256 target = get_kv(ctx, k);257 return true;258 }259 260 static bool set(const gguf_context * ctx, const char * key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {261 return set(ctx, gguf_find_key(ctx, key), target, ovrd);262 }263 264 static bool set(const gguf_context * ctx, const std::string & key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {265 return set(ctx, key.c_str(), target, ovrd);266 }267 };268}269 270 template<typename T>271 typename std::enable_if<std::is_integral<T>::value, bool>::type272 llama_model_loader::get_arr_n(const std::string & key, T & result, bool required) {273 const int kid = gguf_find_key(metadata, key.c_str());274 275 if (kid < 0) {276 if (required) {277 throw std::runtime_error(format("key not found in model: %s", key.c_str()));278 }279 return false;280 }281 282 struct GGUFMeta::ArrayInfo arr_info =283 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(metadata, kid);284 285 286 result = arr_info.length;287 return true;288 }289 290 template<typename T>291 typename std::enable_if<std::is_integral<T>::value, bool>::type292 llama_model_loader::get_arr_n(enum llm_kv kid, T & result, bool required) {293 return get_arr_n(llm_kv(kid), result, required);294 }295 296 template bool llama_model_loader::get_arr_n(enum llm_kv kid, uint32_t & result, bool required);297 298 template<typename T>299 bool llama_model_loader::get_arr(const std::string & key, std::vector<T> & result, bool required) {300 const gguf_context * ctx = metadata;301 const int kid = gguf_find_key(ctx, key.c_str());302 303 if (kid < 0 || gguf_get_kv_type(ctx, kid) != GGUF_TYPE_ARRAY) {304 if (required) {305 throw std::runtime_error(format("array key not found in model: %s", key.c_str()));306 }307 return false;308 }309 310 struct GGUFMeta::ArrayInfo arr_info =311 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx, kid);312 313 switch (arr_info.gt) {314 case GGUF_TYPE_UINT32:315 case GGUF_TYPE_INT32: GGML_ASSERT((std::is_same<T, int32_t>::value) ||316 (std::is_same<T, uint32_t>::value)); break;317 case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value)); break;318 case GGUF_TYPE_STRING: GGML_ASSERT((std::is_same<T, std::string>::value)); break;319 default:320 throw std::runtime_error(format("%s is not a string/float32/uint32/int32 array", key.c_str()));321 }322 323 if constexpr (std::is_same<T, std::string>::value) {324 const size_t n_items = gguf_get_arr_n(ctx, kid);325 result.clear();326 327 for (size_t i = 0; i < n_items; i++) {328 const T value = gguf_get_arr_str(ctx, kid, i);329 result.emplace_back(value);330 }331 } else {332 result.resize(arr_info.length);333 result.assign((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length);334 }335 336 return true;337 }338 339 template<typename T, size_t N_MAX>340 bool llama_model_loader::get_arr(const std::string & key, std::array<T, N_MAX> & result, bool required) {341 const gguf_context * ctx = metadata;342 const int kid = gguf_find_key(ctx, key.c_str());343 344 if (kid < 0 || gguf_get_kv_type(ctx, kid) != GGUF_TYPE_ARRAY) {345 if (required) {346 throw std::runtime_error(format("array key not found in model: %s", key.c_str()));347 }348 return false;349 }350 351 struct GGUFMeta::ArrayInfo arr_info =352 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx, kid);353 354 switch (arr_info.gt) {355 case GGUF_TYPE_BOOL:356 case GGUF_TYPE_UINT32:357 case GGUF_TYPE_INT32: GGML_ASSERT((std::is_same<T, int32_t>::value) ||358 (std::is_same<T, uint32_t>::value)); break;359 case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value)); break;360 case GGUF_TYPE_STRING: GGML_ASSERT((std::is_same<T, std::string>::value)); break;361 default:362 throw std::runtime_error(format("%s is not a string/float32/uint32/int32 array", key.c_str()));363 }364 365 if (arr_info.length > N_MAX) {366 throw std::runtime_error(format("array length %u for key %s exceeds max %u", (uint32_t) arr_info.length, key.c_str(), (uint32_t) N_MAX));367 }368 369 if constexpr (std::is_same<T, std::string>::value) {370 const size_t n_items = gguf_get_arr_n(ctx, kid);371 372 for (size_t i = 0; i < n_items; i++) {373 const T value = gguf_get_arr_str(ctx, kid, i);374 result[i] = value;375 }376 } else {377 if (arr_info.gt == GGUF_TYPE_BOOL) {378 const int8_t * values = (const int8_t *) arr_info.data;379 std::transform(values, values + arr_info.length, result.begin(), [](int8_t x) {380 return static_cast<T>(x != 0);381 });382 } else {383 std::copy((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length, result.begin());384 }385 }386 387 return true;388 }389 390 template<typename T>391 bool llama_model_loader::get_arr(enum llm_kv kid, T & result, bool required) {392 return get_arr(llm_kv(kid), result, required);393 }394 395 template bool llama_model_loader::get_arr<std::vector<std::string>>(enum llm_kv kid, std::vector<std::string> & result, bool required);396 397 template<typename T>398 bool llama_model_loader::get_key(const std::string & key, T & result, bool required) {399 auto it = kv_overrides.find(key);400 401 const struct llama_model_kv_override * override =402 it != kv_overrides.end() ? &it->second : nullptr;403 404 const bool found = GGUFMeta::GKV<T>::set(metadata, key, result, override);405 406 if (required && !found) {407 throw std::runtime_error(format("key not found in model: %s", key.c_str()));408 }409 410 return found;411 }412 413 template<typename T>414 bool llama_model_loader::get_key(enum llm_kv kid, T & result, bool required) {415 return get_key(llm_kv(kid), result, required);416 }417 418 template bool llama_model_loader::get_key<bool> (enum llm_kv kid, bool & result, bool required);419 template bool llama_model_loader::get_key<float> (enum llm_kv kid, float & result, bool required);420 template bool llama_model_loader::get_key<uint32_t> (enum llm_kv kid, uint32_t & result, bool required);421 template bool llama_model_loader::get_key<std::string>(enum llm_kv kid, std::string & result, bool required);422 423 template<>424 bool llama_model_loader::get_key(enum llm_kv kid, enum llama_pooling_type & result, bool required) {425 uint32_t tmp;426 const bool found = get_key(kid, tmp, required);427 if (found) {428 result = (enum llama_pooling_type) tmp;429 } else {430 result = LLAMA_POOLING_TYPE_UNSPECIFIED;431 }432 return found;433 }434 435 // get array of n <= N_MAX elements, or a single element repeated n times436 template<typename T, size_t N_MAX>437 bool llama_model_loader::get_key_or_arr(const std::string & key, std::array<T, N_MAX> & result, uint32_t n, bool required) {438 const int kid = gguf_find_key(metadata, key.c_str());439 440 if (kid < 0) {441 if (required) {442 throw std::runtime_error(format("key not found in model: %s", key.c_str()));443 }444 return false;445 }446 447 if (n > N_MAX) {448 throw std::runtime_error(format("n > N_MAX: %u > %u for key %s", (uint32_t) n, (uint32_t) N_MAX, key.c_str()));449 }450 451 if (gguf_get_kv_type(metadata, kid) == GGUF_TYPE_ARRAY) {452 struct GGUFMeta::ArrayInfo arr_info =453 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(metadata, kid);454 455 if (n != arr_info.length) {456 throw std::runtime_error(format("key %s has wrong array length; expected %u, got %u", key.c_str(), n, (uint32_t) arr_info.length));457 }458 459 return get_arr(key, result, required);460 }461 462 T value;463 464 bool ok = get_key(key, value, required);465 if (!ok) {466 return false;467 }468 469 for (uint32_t i = 0; i < n; i++) {470 result[i] = value;471 }472 473 return true;474 }475 476 template<typename T>477 bool llama_model_loader::get_key_or_arr(enum llm_kv kid, T & result, uint32_t n, bool required) {478 return get_key_or_arr(llm_kv(kid), result, n, required);479 }480 481 bool llama_model_loader::get_key_or_arr(enum llm_kv kid, uint32_t & result, bool required) {482 const std::string key = llm_kv(kid);483 484 const int id = gguf_find_key(metadata, key.c_str());485 486 if (id < 0) {487 if (required) {488 throw std::runtime_error(format("key not found in model: %s", key.c_str()));489 }490 return false;491 }492 493 // throw and error if type is an array494 if (gguf_get_kv_type(metadata, id) == GGUF_TYPE_ARRAY) {495 if (required) {496 throw std::runtime_error(format("expected scalar, found array for key: %s", key.c_str()));497 }498 return false;499 }500 501 return get_key(key, result, required);502 }503 504 // TODO: this is not very clever - figure out something better505 template bool llama_model_loader::get_key_or_arr<std::array<int, 4>>(enum llm_kv kid, std::array<int, 4> & result, uint32_t n, bool required);506 template bool llama_model_loader::get_key_or_arr<std::array<uint32_t, 512>>(enum llm_kv kid, std::array<uint32_t, 512> & result, uint32_t n, bool required);507 template bool llama_model_loader::get_key_or_arr<std::array<float, 512>>(enum llm_kv kid, std::array<float, 512> & result, uint32_t n, bool required);508 509 510llama_model_loader::llama_model_loader(511 struct gguf_context * meta,512 llama_model_set_tensor_data_t set_tensor_data,513 void * set_tensor_data_ud,514 const std::string & fname,515 std::vector<std::string> & splits,516 FILE * file,517 bool use_mmap,518 bool use_direct_io,519 bool check_tensors,520 bool no_alloc,521 const llama_model_kv_override * param_overrides_p,522 const llama_model_tensor_buft_override * param_tensor_buft_overrides_p)523 : metadata(meta), set_tensor_data(set_tensor_data), set_tensor_data_ud(set_tensor_data_ud) {524 int trace = 0;525 if (getenv("LLAMA_TRACE")) {526 trace = atoi(getenv("LLAMA_TRACE"));527 }528 529 if (param_overrides_p != nullptr) {530 for (const struct llama_model_kv_override * p = param_overrides_p; p->key[0] != 0; p++) {531 kv_overrides.insert({std::string(p->key), *p});532 }533 }534 535 tensor_buft_overrides = param_tensor_buft_overrides_p;536 537 if (!fname.empty()) {538 // Load the main GGUF539 struct ggml_context * ctx = NULL;540 struct gguf_init_params params = {541 /*.no_alloc = */ true,542 /*.ctx = */ &ctx,543 };544 545 metadata_ptr.reset(gguf_init_from_file(fname.c_str(), params));546 metadata = metadata_ptr.get();547 if (metadata == nullptr) {548 throw std::runtime_error(format("%s: failed to load model from %s", __func__, fname.c_str()));549 }550 551 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);552 llm_kv = LLM_KV(llm_arch_from_string(arch_name));553 554 files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io));555 contexts.emplace_back(ctx);556 557 if (use_mmap && use_direct_io) {558 if (files.back()->has_direct_io()) {559 LLAMA_LOG_WARN("%s: direct I/O is enabled, disabling mmap\n", __func__);560 use_mmap = false;561 } else {562 LLAMA_LOG_WARN("%s: direct I/O is not available, using mmap\n", __func__);563 use_direct_io = false;564 565 // reopen file using std::fopen for mmap566 files.pop_back();567 files.emplace_back(new llama_file(fname.c_str(), "rb", false));568 }569 }570 571 // Save tensors data offset of the main file.572 // For subsidiary files, `meta` tensor data offset must not be used,573 // so we build a unified tensors index for weights.574 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {575 std::string tensor_name = std::string(cur->name);576 // make sure there is no duplicated tensor names577 if (weights_map.find(tensor_name) != weights_map.end()) {578 throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));579 }580 n_elements += ggml_nelements(cur);581 n_bytes += ggml_nbytes(cur);582 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur));583 }584 uint16_t n_split = 0;585 get_key(llm_kv(LLM_KV_SPLIT_COUNT), n_split, false);586 587 // Load additional GGML contexts588 if (n_split > 1) {589 // make sure the main file is loaded first590 uint16_t idx = 0;591 const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO);592 get_key(kv_split_no, idx);593 if (idx != 0) {594 throw std::runtime_error(format("illegal split file idx: %d (file: %s), model must be loaded with the first split", idx, fname.c_str()));595 }596 597 // generate list of splits if needed598 if (splits.empty()) {599 splits = llama_get_list_splits(fname, idx, n_split);600 }601 602 // in case user give a custom list of splits, check if it matches the expected number603 if (n_split != (uint16_t)splits.size()) {604 throw std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", splits.size(), n_split));605 }606 607 if (trace > 0) {608 LLAMA_LOG_INFO("%s: loading additional %d GGUFs\n", __func__, n_split);609 }610 611 // load other splits612 for (idx = 1; idx < n_split; idx++) {613 const char * fname_split = splits[idx].c_str();614 615 struct gguf_init_params split_params = {616 /*.no_alloc = */ true,617 /*.ctx = */ &ctx,618 };619 gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) };620 if (!ctx_gguf) {621 throw std::runtime_error(format("%s: failed to load GGUF split from %s", __func__, fname_split));622 }623 624 // check idx625 {626 const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str());627 if (kid < 0) {628 throw std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split));629 }630 int idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid);631 if (idx_gguf != idx) {632 throw std::runtime_error(format("invalid split file idx: %d (file: %s), expected %d", idx_gguf, fname_split, idx));633 }634 }635 636 files.emplace_back(new llama_file(fname_split, "rb", use_direct_io));637 contexts.emplace_back(ctx);638 639 // Save tensors data offset info of the shard.640 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {641 std::string tensor_name = std::string(cur->name);642 // make sure there is no duplicated tensor names643 if (weights_map.find(tensor_name) != weights_map.end()) {644 throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));645 }646 n_elements += ggml_nelements(cur);647 n_bytes += ggml_nbytes(cur);648 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), idx, ctx_gguf.get(), cur));649 }650 }651 652 get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), n_tensors);653 654 // sanity check655 {656 const int n_tensors_loaded = (int) weights_map.size();657 if (n_tensors != n_tensors_loaded) {658 throw std::runtime_error(format("corrupted model: %d tensors expected but %d found", n_tensors, n_tensors_loaded));659 }660 }661 662 LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split - 1);663 }664 } else if (file != nullptr) {665 struct ggml_context * ctx = NULL;666 struct gguf_init_params params = {667 /*.no_alloc = */ true,668 /*.ctx = */ &ctx,669 };670 671 metadata_ptr.reset(gguf_init_from_file_ptr(file, params));672 metadata = metadata_ptr.get();673 if (metadata == nullptr) {674 throw std::runtime_error(format("%s: failed to load model from file pointer", __func__));675 }676 677 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);678 llm_kv = LLM_KV(llm_arch_from_string(arch_name));679 680 files.emplace_back(new llama_file(file));681 contexts.emplace_back(ctx);682 683 // Save tensors data offset info of the main file.684 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {685 std::string tensor_name = std::string(cur->name);686 // make sure there is no duplicated tensor names687 if (weights_map.find(tensor_name) != weights_map.end()) {688 throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));689 }690 n_elements += ggml_nelements(cur);691 n_bytes += ggml_nbytes(cur);692 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur));693 }694 } else {695 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);696 llm_kv = LLM_KV(llm_arch_from_string(arch_name));697 }698 699 n_kv = gguf_get_n_kv(metadata);700 n_tensors = weights_map.size();701 702 fver = (enum llama_fver) gguf_get_version(metadata);703 704 LLAMA_LOG_INFO("%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n",705 __func__, n_kv, n_tensors, fname.empty() ? "(file*)" : fname.c_str(), llama_file_version_name(fver));706 707 // determine file type based on the number of tensors for each quantization and print meta data708 // TODO: make optional709 {710 std::map<enum ggml_type, uint32_t> n_type;711 712 uint32_t n_type_max = 0;713 enum ggml_type type_max = GGML_TYPE_F32;714 715 for (const auto & it : weights_map) {716 const llama_tensor_weight & w = it.second;717 const ggml_tensor * tensor = w.tensor;718 719 enum ggml_type type = tensor->type;720 721 n_type[type]++;722 723 if (n_type_max < n_type[type]) {724 n_type_max = n_type[type];725 type_max = type;726 }727 728 if (trace > 0) {729 const uint16_t sid = w.idx;730 LLAMA_LOG_INFO("%s: - tensor split %2d: %32s %-8s [ %s ] %8.2f MiB\n", __func__,731 sid, ggml_get_name(tensor), ggml_type_name(type), llama_format_tensor_shape(tensor).c_str(),732 ggml_nbytes(tensor)/1024.0f/1024.0f);733 }734 }735 736 switch (type_max) {737 case GGML_TYPE_F32: ftype = LLAMA_FTYPE_ALL_F32; break;738 case GGML_TYPE_F16: ftype = LLAMA_FTYPE_MOSTLY_F16; break;739 case GGML_TYPE_BF16: ftype = LLAMA_FTYPE_MOSTLY_BF16; break;740 case GGML_TYPE_Q4_0: ftype = LLAMA_FTYPE_MOSTLY_Q4_0; break;741 case GGML_TYPE_Q4_1: ftype = LLAMA_FTYPE_MOSTLY_Q4_1; break;742 case GGML_TYPE_Q5_0: ftype = LLAMA_FTYPE_MOSTLY_Q5_0; break;743 case GGML_TYPE_Q5_1: ftype = LLAMA_FTYPE_MOSTLY_Q5_1; break;744 case GGML_TYPE_Q8_0: ftype = LLAMA_FTYPE_MOSTLY_Q8_0; break;745 case GGML_TYPE_Q2_K: ftype = LLAMA_FTYPE_MOSTLY_Q2_K; break;746 case GGML_TYPE_Q3_K: ftype = LLAMA_FTYPE_MOSTLY_Q3_K_M; break;747 case GGML_TYPE_Q4_K: ftype = LLAMA_FTYPE_MOSTLY_Q4_K_M; break;748 case GGML_TYPE_Q5_K: ftype = LLAMA_FTYPE_MOSTLY_Q5_K_M; break;749 case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break;750 case GGML_TYPE_TQ1_0: ftype = LLAMA_FTYPE_MOSTLY_TQ1_0; break;751 case GGML_TYPE_TQ2_0: ftype = LLAMA_FTYPE_MOSTLY_TQ2_0; break;752 case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break;753 case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break;754 case GGML_TYPE_IQ2_S: ftype = LLAMA_FTYPE_MOSTLY_IQ2_S; break;755 case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break;756 case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break;757 case GGML_TYPE_IQ1_M: ftype = LLAMA_FTYPE_MOSTLY_IQ1_M; break;758 case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break;759 case GGML_TYPE_IQ4_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ4_XS; break;760 case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break;761 case GGML_TYPE_NVFP4: ftype = LLAMA_FTYPE_MOSTLY_NVFP4; break;762 case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break;763 default:764 {765 LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));766 ftype = LLAMA_FTYPE_ALL_F32;767 } break;768 }769 770 // this is a way to mark that we have "guessed" the file type771 ftype = (llama_ftype) (ftype | LLAMA_FTYPE_GUESSED);772 773 {774 uint32_t ftype_val = 0;775 if (get_key(LLM_KV_GENERAL_FILE_TYPE, ftype_val, false)) {776 ftype = (llama_ftype) ftype_val;777 }778 }779 780 LLAMA_LOG_INFO("%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", __func__);781 782 for (int i = 0; i < n_kv; i++) {783 const char * name = gguf_get_key(metadata, i);784 const enum gguf_type type = gguf_get_kv_type(metadata, i);785 const std::string type_name =786 type == GGUF_TYPE_ARRAY787 ? format("%s[%s,%zu]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(metadata, i)), gguf_get_arr_n(metadata, i))788 : gguf_type_name(type);789 790 std::string value = gguf_kv_to_str(metadata, i);791 const size_t MAX_VALUE_LEN = 40;792 if (value.size() > MAX_VALUE_LEN) {793 value = format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str());794 }795 replace_all(value, "\n", "\\n");796 797 LLAMA_LOG_INFO("%s: - kv %3d: %42s %-16s = %s\n", __func__, i, name, type_name.c_str(), value.c_str());798 }799 800 // print type counts801 for (auto & kv : n_type) {802 if (kv.second == 0) {803 continue;804 }805 806 LLAMA_LOG_INFO("%s: - type %4s: %4d tensors\n", __func__, ggml_type_name(kv.first), kv.second);807 }808 }809 810 if (!llama_mmap::SUPPORTED) {811 LLAMA_LOG_WARN("%s: mmap is not supported on this platform\n", __func__);812 use_mmap = false;813 }814 815 this->use_mmap = use_mmap;816 this->use_direct_io = use_direct_io;817 this->check_tensors = check_tensors;818 this->no_alloc = no_alloc;819}820 821std::string llama_model_loader::get_arch_name() const {822 return arch_name;823}824 825enum llm_arch llama_model_loader::get_arch() const {826 return llm_kv.arch;827}828 829const llama_model_loader::llama_tensor_weight * llama_model_loader::get_weight(const char * name) const {830 auto pos = weights_map.find(name);831 if (pos != weights_map.end()) {832 return &pos->second;833 }834 835 return nullptr;836}837 838const llama_model_loader::llama_tensor_weight & llama_model_loader::require_weight(const char * name) const {839 const llama_tensor_weight * weight = get_weight(name);840 if (!weight) {841 throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name));842 }843 return *weight;844}845 846struct ggml_tensor * llama_model_loader::get_tensor_meta(const char * name) const {847 const auto * weight = get_weight(name);848 if (!weight) {849 return nullptr;850 }851 return weight->tensor;852}853 854struct ggml_tensor * llama_model_loader::require_tensor_meta(const std::string & name) const {855 struct ggml_tensor * tensor = get_tensor_meta(name.c_str());856 if (!tensor) {857 throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));858 }859 return tensor;860}861 862const struct ggml_tensor * llama_model_loader::check_tensor_dims(const std::string & name, const std::vector<int64_t> & ne, bool required) const {863 const struct ggml_tensor * cur = get_tensor_meta(name.c_str());864 865 if (cur == NULL) {866 if (!required) {867 return NULL;868 }869 throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));870 }871 872 {873 bool is_ok = true;874 for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {875 if ((i < ne.size() && ne[i] != cur->ne[i]) || (i >= ne.size() && cur->ne[i] != 1)) {876 is_ok = false;877 break;878 }879 }880 if (!is_ok) {881 throw std::runtime_error(882 format("%s: tensor '%s' has wrong shape; expected %s, got %s",883 __func__, name.c_str(),884 llama_format_tensor_shape(ne).c_str(),885 llama_format_tensor_shape(cur).c_str()));886 }887 }888 889 return cur;890}891 892// checks if the weight tensor can be used with the specified buffer type and device893static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w, ggml_op op, ggml_backend_buffer_type_t buft, ggml_backend_dev_t dev) {894 GGML_ASSERT(w != nullptr);895 896 if (op == GGML_OP_NONE) {897 return true;898 }899 900 ggml_init_params params = {901 /*.mem_size =*/ ggml_tensor_overhead()*8,902 /*.mem_buffer =*/ NULL,903 /*.no_alloc =*/ true,904 };905 ggml_context_ptr ctx_ptr { ggml_init(params) };906 if (!ctx_ptr) {907 throw std::runtime_error(format("failed to create ggml context"));908 }909 ggml_context * ctx = ctx_ptr.get();910 911 ggml_tensor * op_tensor = nullptr;912 913 switch (op) {914 case GGML_OP_GET_ROWS:915 {916 ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);917 op_tensor = ggml_get_rows(ctx, w, b);918 } break;919 case GGML_OP_MUL_MAT:920 {921 ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], 512, w->ne[2], w->ne[3]);922 op_tensor = ggml_mul_mat(ctx, w, b);923 } break;924 case GGML_OP_MUL_MAT_ID:925 {926 const int n_expert_used = hparams.n_expert_used;927 GGML_ASSERT(n_expert_used > 0);928 ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);929 ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);930 op_tensor = ggml_mul_mat_id(ctx, w, b, ids);931 } break;932 case GGML_OP_ADD:933 {934 ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], w->ne[1], w->ne[2], w->ne[3]);935 op_tensor = ggml_add(ctx, a, w);936 } break;937 case GGML_OP_ADD_ID:938 {939 const int n_expert_used = hparams.n_expert_used;940 GGML_ASSERT(n_expert_used > 0);941 ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);942 ggml_tensor * c = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);943 op_tensor = ggml_add_id(ctx, a, w, c);944 } break;945 case GGML_OP_MUL:946 {947 ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], w->ne[1], w->ne[2], w->ne[3]);948 op_tensor = ggml_mul(ctx, a, w);949 } break;950 case GGML_OP_DIV:951 {952 ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, w->ne[0]);953 op_tensor = ggml_div(ctx, a, w);954 } break;955 case GGML_OP_ROPE:956 {957 const int n_embd_head = hparams.n_embd_head_v();958 const int n_head = hparams.n_head();959 ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_head, n_head, 512);960 ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);961 op_tensor = ggml_rope_ext(962 ctx, a, b, w,963 0, 0, 0, 0, 0,964 0, 0, 0, 0965 );966 967 } break;968 case GGML_OP_SSM_CONV:969 {970 const int64_t n_seq_tokens = 512;971 const int64_t n_seqs = 3;972 ggml_tensor * conv_x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0] - 1 + n_seq_tokens, w->ne[1], n_seqs);973 op_tensor = ggml_ssm_conv(ctx, conv_x, w);974 } break;975 case GGML_OP_SSM_SCAN:976 {977 // w is ssm_a, which is used to distinguish Mamba-1 and Mamba-2978 const int64_t d_state = w->ne[0] == 1 ? hparams.ssm_d_state : w->ne[0];979 const int64_t n_head = w->ne[1];980 const int64_t head_dim = hparams.ssm_d_inner / n_head;981 const int64_t n_group = hparams.ssm_n_group ? hparams.ssm_n_group : 1;982 const int64_t n_seq_tokens = 512;983 const int64_t n_seqs = 3;984 ggml_tensor * s = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, head_dim, n_head, n_seqs);985 ggml_tensor * x = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, head_dim, n_head, n_seq_tokens, n_seqs);986 ggml_tensor * dt = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_head, n_seq_tokens, n_seqs);987 ggml_tensor * B = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, n_group, n_seq_tokens, n_seqs);988 ggml_tensor * C = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, n_group, n_seq_tokens, n_seqs);989 ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_seqs);990 op_tensor = ggml_ssm_scan(ctx, s, x, dt, w, B, C, ids);991 } break;992 case GGML_OP_RWKV_WKV6:993 {994 // FIXME995 const int64_t S = 123;996 const int64_t H = 123;997 const int64_t n_tokens = 123;998 const int64_t n_seqs = 123;999 ggml_tensor * k = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);1000 ggml_tensor * v = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);1001 ggml_tensor * r = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);1002 ggml_tensor * tf = w;1003 ggml_tensor * td = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);1004 ggml_tensor * state = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, S, n_seqs, S, H);1005 op_tensor = ggml_rwkv_wkv6(ctx, k, v, r, tf, td, state);1006 } break;1007 case GGML_OP_IM2COL:1008 {1009 const int n_embd_inp = hparams.n_embd_inp();1010 ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_embd_inp, w->ne[1], 1, 1);1011 op_tensor = ggml_im2col(ctx, w, b, 1, 0, 0, 0, 1, 0, false, GGML_TYPE_F16);1012 } break;1013 case GGML_OP_SCALE:1014 {1015 op_tensor = ggml_scale(ctx, w, 1.0f);1016 } break;1017 default:1018 GGML_ABORT("%s: missing test for op %s for tensor %s", __func__, ggml_op_name(op), w->name);1019 }1020 1021 // create a temporary dummy buffer for the weight so that supports_op can check the buffer type1022 GGML_ASSERT(w->buffer == nullptr);1023 w->buffer = ggml_backend_buft_alloc_buffer(buft, 0);1024 bool op_supported = ggml_backend_dev_supports_op(dev, op_tensor);1025 ggml_backend_buffer_free(w->buffer);1026 w->buffer = nullptr;1027 1028 return op_supported;1029}1030 1031// find the first buffer type in the list that can use the tensor1032static ggml_backend_buffer_type_t select_weight_buft(const llama_hparams & hparams, ggml_tensor * tensor, ggml_op op, const buft_list_t * buft_list) {1033 GGML_ASSERT(!buft_list->empty());1034 for (const auto & cur : *buft_list) {1035 ggml_backend_dev_t cur_dev = cur.first;1036 ggml_backend_buffer_type_t cur_buft = cur.second;1037 if (weight_buft_supported(hparams, tensor, op, cur_buft, cur_dev)) {1038 return cur_buft;1039 }1040 }1041 1042 return nullptr;1043}1044 1045struct ggml_tensor * llama_model_loader::create_tensor(1046 const llama_hparams & hparams, const buft_list_t * buft_list_cpu, const buft_list_t * buft_list_input, const buft_list_t * buft_list_output,1047 const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {1048 auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {1049 auto it = ctx_map.find(buft);1050 if (it == ctx_map.end()) {1051 // one ggml context per buffer type1052 int max_n_tensors = n_tensors;1053 max_n_tensors += 1; // duplicated output tensor1054 max_n_tensors += hparams.n_layer*2; // duplicated rope freq tensors1055 if (files.empty()) {1056 max_n_tensors += hparams.n_layer*256; // this should be well above what any model actually uses1057 }1058 const size_t ctx_size = ggml_tensor_overhead()*max_n_tensors;1059 1060 ggml_init_params params = {1061 /*.mem_size =*/ ctx_size,1062 /*.mem_buffer =*/ NULL,1063 /*.no_alloc =*/ true,1064 };1065 1066 ggml_context * ctx = ggml_init(params);1067 if (!ctx) {1068 throw std::runtime_error(format("failed to create ggml context"));1069 }1070 1071 ctx_map.emplace(buft, ctx);1072 1073 return ctx;1074 }1075 return it->second.get();1076 };1077 1078 auto buft_for_tensor = [&](ggml_tensor * t_meta) -> ggml_backend_buffer_type_t {1079 if (!t_meta) {1080 if (flags & TENSOR_NOT_REQUIRED) {1081 return nullptr;1082 }1083 throw std::runtime_error(format("missing tensor '%s'", tn.str().c_str()));1084 }1085 1086 // some models use the token embedding tensor as the output, but since these are used in different layers and with different ops1087 // the tensor is duplicated1088 // to handle this, we check if the tensor is duplicated, and if so, we assume that it is being loaded as the output tensor1089 llm_tensor tn_tensor = tn.tensor;1090 if (tn.tensor == LLM_TENSOR_TOKEN_EMBD && (flags & TENSOR_DUPLICATED)) {1091 tn_tensor = LLM_TENSOR_OUTPUT;1092 }1093 1094 llm_tensor_info info;1095 try {1096 info = llm_tensor_info_for(tn_tensor);1097 } catch (const std::out_of_range & e) {1098 throw std::runtime_error(format("missing tensor info mapping for %s", tn.str().c_str()));1099 }1100 1101 // skip unused tensors1102 if (info.op == GGML_OP_NONE || (flags & TENSOR_SKIP)) {1103 const size_t nbytes = ggml_nbytes(t_meta);1104 LLAMA_LOG_WARN("model has unused tensor %s (size = %zu bytes) -- ignoring\n", tn.str().c_str(), nbytes);1105 1106 size_data -= nbytes;1107 n_created++;1108 1109 return nullptr;1110 }1111 1112 // tensors with "bias" suffix are always used with GGML_OP_ADD or GGML_OP_ADD_ID1113 ggml_op op;1114 bool bias = tn.suffix != nullptr && strcmp(tn.suffix, "bias") == 0;1115 if (bias) {1116 if (info.op == GGML_OP_MUL_MAT_ID) {1117 op = GGML_OP_ADD_ID;1118 } else {1119 op = GGML_OP_ADD;1120 }1121 } else {1122 op = info.op;1123 }1124 1125 // sanity checks1126 if (info.layer == LLM_TENSOR_LAYER_INPUT || info.layer == LLM_TENSOR_LAYER_OUTPUT) {1127 if (tn.bid != -1) {1128 GGML_ABORT("input/output layer tensor %s used with a layer number", tn.str().c_str());1129 }1130 } else {1131 if (tn.bid == -1) {1132 GGML_ABORT("repeating layer tensor %s used without a layer number", tn.str().c_str());1133 }1134 }1135 1136 // select the buffer type for this tensor1137 const buft_list_t * buft_list;1138 switch (info.layer) {1139 case LLM_TENSOR_LAYER_INPUT:1140 buft_list = buft_list_input;1141 break;1142 case LLM_TENSOR_LAYER_OUTPUT:1143 buft_list = buft_list_output;1144 break;1145 case LLM_TENSOR_LAYER_REPEATING:1146 GGML_ASSERT(buft_list_layer != nullptr);1147 buft_list = buft_list_layer;1148 break;1149 default:1150 GGML_ABORT("invalid layer %d for tensor %s", info.layer, tn.str().c_str());1151 }1152 1153 ggml_backend_buffer_type_t buft = nullptr;1154 1155 // check overrides1156 if (tensor_buft_overrides) {1157 std::string tensor_name = tn.str();1158 for (const auto * overrides = tensor_buft_overrides; overrides->pattern != nullptr; ++overrides) {1159 std::regex pattern(overrides->pattern);1160 if (std::regex_search(tensor_name, pattern)) {1161 if (overrides->buft == ggml_backend_cpu_buffer_type()) {1162 // when overriding to a CPU buffer, consider the extra buffer types1163 buft = select_weight_buft(hparams, t_meta, op, buft_list_cpu);1164 if (use_mmap) {1165 static std::once_flag once;1166 std::call_once(once, [] {1167 LLAMA_LOG_WARN("llama_model_loader: tensor overrides to CPU are used with mmap enabled - consider using --no-mmap for better performance\n");1168 });1169 }1170 } else {1171 buft = overrides->buft;1172 }1173 1174 LLAMA_LOG_DEBUG("tensor %s (%zu MiB %s) buffer type overridden to %s\n",1175 tensor_name.c_str(),1176 ggml_nbytes(t_meta) / 1024 / 1024, ggml_type_name(t_meta->type),1177 ggml_backend_buft_name(buft));1178 break;1179 }1180 }1181 }1182 1183 if (!buft) {1184 buft = select_weight_buft(hparams, t_meta, op, buft_list);1185 if (!buft) {1186 throw std::runtime_error(format("failed to find a compatible buffer type for tensor %s", tn.str().c_str()));1187 }1188 }1189 1190 // avoid using a host buffer when using mmap1191 auto * buft_dev = ggml_backend_buft_get_device(buft);1192 if (use_mmap && buft_dev && buft == ggml_backend_dev_host_buffer_type(buft_dev)) {1193 auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);1194 if (!cpu_dev) {1195 throw std::runtime_error("no CPU backend found");1196 }1197 buft = ggml_backend_dev_buffer_type(cpu_dev);1198 }1199 1200 if (buft != buft_list->front().second) {