KBaba7/llama.cpp
0
1#include "llama-model-loader.h"2 3#include "ggml.h"4 5#include <array>6#include <cinttypes>7#include <cstring>8#include <future>9 10static const size_t kiB = 1024;11static const size_t MiB = 1024*kiB;12static const size_t GiB = 1024*MiB;13 14const char * llama_file_version_name(llama_fver version) {15 switch (version) {16 case GGUF_FILE_VERSION_V1: return "GGUF V1 (support until nov 2023)";17 case GGUF_FILE_VERSION_V2: return "GGUF V2";18 case GGUF_FILE_VERSION_V3: return "GGUF V3 (latest)";19 }20 21 return "unknown";22}23 24static std::string llama_model_ftype_name(llama_ftype ftype) {25 if (ftype & LLAMA_FTYPE_GUESSED) {26 return llama_model_ftype_name((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) + " (guessed)";27 }28 29 switch (ftype) {30 case LLAMA_FTYPE_ALL_F32: return "all F32";31 case LLAMA_FTYPE_MOSTLY_F16: return "F16";32 case LLAMA_FTYPE_MOSTLY_BF16: return "BF16";33 case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0";34 case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1";35 case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0";36 case LLAMA_FTYPE_MOSTLY_Q5_1: return "Q5_1";37 case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0";38 case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K - Medium";39 case LLAMA_FTYPE_MOSTLY_Q2_K_S: return "Q2_K - Small";40 case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small";41 case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium";42 case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large";43 case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "Q4_K - Small";44 case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "Q4_K - Medium";45 case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small";46 case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium";47 case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K";48 case LLAMA_FTYPE_MOSTLY_TQ1_0: return "TQ1_0 - 1.69 bpw ternary";49 case LLAMA_FTYPE_MOSTLY_TQ2_0: return "TQ2_0 - 2.06 bpw ternary";50 case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return "IQ2_XXS - 2.0625 bpw";51 case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw";52 case LLAMA_FTYPE_MOSTLY_IQ2_S: return "IQ2_S - 2.5 bpw";53 case LLAMA_FTYPE_MOSTLY_IQ2_M: return "IQ2_M - 2.7 bpw";54 case LLAMA_FTYPE_MOSTLY_IQ3_XS: return "IQ3_XS - 3.3 bpw";55 case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return "IQ3_XXS - 3.0625 bpw";56 case LLAMA_FTYPE_MOSTLY_IQ1_S: return "IQ1_S - 1.5625 bpw";57 case LLAMA_FTYPE_MOSTLY_IQ1_M: return "IQ1_M - 1.75 bpw";58 case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw";59 case LLAMA_FTYPE_MOSTLY_IQ4_XS: return "IQ4_XS - 4.25 bpw";60 case LLAMA_FTYPE_MOSTLY_IQ3_S: return "IQ3_S - 3.4375 bpw";61 case LLAMA_FTYPE_MOSTLY_IQ3_M: return "IQ3_S mix - 3.66 bpw";62 63 default: return "unknown, may not work";64 }65}66 67// return a list of splits for a given path68// for example, given "<name>-00002-of-00004.gguf", returns list of all 4 splits69static std::vector<std::string> llama_get_list_splits(const std::string & path, const int idx, const int n_split) {70 std::vector<std::string> paths;71 std::string split_prefix;72 std::vector<char> buf(llama_path_max(), 0);73 74 {75 int ret = llama_split_prefix(buf.data(), buf.size(), path.c_str(), idx, n_split);76 if (!ret) {77 throw std::runtime_error(format("invalid split file name: %s", path.c_str()));78 }79 split_prefix = std::string(buf.data(), ret);80 }81 82 if (split_prefix.empty()) {83 throw std::runtime_error(format("invalid split file: %s", path.c_str()));84 }85 86 for (int idx = 0; idx < n_split; ++idx) {87 int ret = llama_split_path(buf.data(), buf.size(), split_prefix.c_str(), idx, n_split);88 paths.push_back(std::string(buf.data(), ret));89 }90 91 return paths;92}93 94namespace GGUFMeta {95 template <typename T, gguf_type gt_, T (*gfun)(const gguf_context *, const int64_t)>96 struct GKV_Base_Type {97 static constexpr gguf_type gt = gt_;98 99 static T getter(const gguf_context * ctx, const int kid) {100 return gfun(ctx, kid);101 }102 };103 104 template<typename T> struct GKV_Base;105 106 template<> struct GKV_Base<bool >: GKV_Base_Type<bool, GGUF_TYPE_BOOL, gguf_get_val_bool> {};107 template<> struct GKV_Base<uint8_t >: GKV_Base_Type<uint8_t, GGUF_TYPE_UINT8, gguf_get_val_u8 > {};108 template<> struct GKV_Base<uint16_t >: GKV_Base_Type<uint16_t, GGUF_TYPE_UINT16, gguf_get_val_u16 > {};109 template<> struct GKV_Base<uint32_t >: GKV_Base_Type<uint32_t, GGUF_TYPE_UINT32, gguf_get_val_u32 > {};110 template<> struct GKV_Base<uint64_t >: GKV_Base_Type<uint64_t, GGUF_TYPE_UINT64, gguf_get_val_u64 > {};111 template<> struct GKV_Base<int8_t >: GKV_Base_Type<int8_t, GGUF_TYPE_INT8, gguf_get_val_i8 > {};112 template<> struct GKV_Base<int16_t >: GKV_Base_Type<int16_t, GGUF_TYPE_INT16, gguf_get_val_i16 > {};113 template<> struct GKV_Base<int32_t >: GKV_Base_Type<int32_t, GGUF_TYPE_INT32, gguf_get_val_i32 > {};114 template<> struct GKV_Base<int64_t >: GKV_Base_Type<int64_t, GGUF_TYPE_INT64, gguf_get_val_i64 > {};115 template<> struct GKV_Base<float >: GKV_Base_Type<float, GGUF_TYPE_FLOAT32, gguf_get_val_f32 > {};116 template<> struct GKV_Base<double >: GKV_Base_Type<double, GGUF_TYPE_FLOAT64, gguf_get_val_f64 > {};117 template<> struct GKV_Base<const char *>: GKV_Base_Type<const char *, GGUF_TYPE_STRING, gguf_get_val_str > {};118 119 template<> struct GKV_Base<std::string> {120 static constexpr gguf_type gt = GGUF_TYPE_STRING;121 122 static std::string getter(const gguf_context * ctx, const int kid) {123 return gguf_get_val_str(ctx, kid);124 }125 };126 127 struct ArrayInfo {128 const gguf_type gt;129 const size_t length;130 const void * data;131 };132 133 template<> struct GKV_Base<ArrayInfo> {134 public:135 static constexpr gguf_type gt = GGUF_TYPE_ARRAY;136 static ArrayInfo getter(const gguf_context *ctx, const int k) {137 const enum gguf_type arr_type = gguf_get_arr_type(ctx, k);138 return ArrayInfo {139 arr_type,140 size_t(gguf_get_arr_n(ctx, k)),141 arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx, k),142 };143 }144 };145 146 template<typename T>147 class GKV : public GKV_Base<T> {148 GKV() = delete;149 150 public:151 static T get_kv(const gguf_context * ctx, const int k) {152 const enum gguf_type kt = gguf_get_kv_type(ctx, k);153 154 if (kt != GKV::gt) {155 throw std::runtime_error(format("key %s has wrong type %s but expected type %s",156 gguf_get_key(ctx, k), gguf_type_name(kt), gguf_type_name(GKV::gt)));157 }158 return GKV::getter(ctx, k);159 }160 161 static const char * override_type_to_str(const llama_model_kv_override_type ty) {162 switch (ty) {163 case LLAMA_KV_OVERRIDE_TYPE_BOOL: return "bool";164 case LLAMA_KV_OVERRIDE_TYPE_INT: return "int";165 case LLAMA_KV_OVERRIDE_TYPE_FLOAT: return "float";166 case LLAMA_KV_OVERRIDE_TYPE_STR: return "str";167 }168 return "unknown";169 }170 171 static bool validate_override(const llama_model_kv_override_type expected_type, const struct llama_model_kv_override * ovrd) {172 if (!ovrd) { return false; }173 if (ovrd->tag == expected_type) {174 LLAMA_LOG_INFO("%s: Using metadata override (%5s) '%s' = ",175 __func__, override_type_to_str(ovrd->tag), ovrd->key);176 switch (ovrd->tag) {177 case LLAMA_KV_OVERRIDE_TYPE_BOOL: {178 LLAMA_LOG_INFO("%s\n", ovrd->val_bool ? "true" : "false");179 } break;180 case LLAMA_KV_OVERRIDE_TYPE_INT: {181 LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->val_i64);182 } break;183 case LLAMA_KV_OVERRIDE_TYPE_FLOAT: {184 LLAMA_LOG_INFO("%.6f\n", ovrd->val_f64);185 } break;186 case LLAMA_KV_OVERRIDE_TYPE_STR: {187 LLAMA_LOG_INFO("%s\n", ovrd->val_str);188 } break;189 default:190 // Shouldn't be possible to end up here, but just in case...191 throw std::runtime_error(192 format("Unsupported attempt to override %s type for metadata key %s\n",193 override_type_to_str(ovrd->tag), ovrd->key));194 }195 return true;196 }197 LLAMA_LOG_WARN("%s: Warning: Bad metadata override type for key '%s', expected %s but got %s\n",198 __func__, ovrd->key, override_type_to_str(expected_type), override_type_to_str(ovrd->tag));199 return false;200 }201 202 template<typename OT>203 static typename std::enable_if<std::is_same<OT, bool>::value, bool>::type204 try_override(OT & target, const struct llama_model_kv_override * ovrd) {205 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, ovrd)) {206 target = ovrd->val_bool;207 return true;208 }209 return false;210 }211 212 template<typename OT>213 static typename std::enable_if<!std::is_same<OT, bool>::value && std::is_integral<OT>::value, bool>::type214 try_override(OT & target, const struct llama_model_kv_override * ovrd) {215 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, ovrd)) {216 target = ovrd->val_i64;217 return true;218 }219 return false;220 }221 222 template<typename OT>223 static typename std::enable_if<std::is_floating_point<OT>::value, bool>::type224 try_override(T & target, const struct llama_model_kv_override * ovrd) {225 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, ovrd)) {226 target = ovrd->val_f64;227 return true;228 }229 return false;230 }231 232 template<typename OT>233 static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::type234 try_override(T & target, const struct llama_model_kv_override * ovrd) {235 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_STR, ovrd)) {236 target = ovrd->val_str;237 return true;238 }239 return false;240 }241 242 static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override * ovrd = nullptr) {243 if (try_override<T>(target, ovrd)) {244 return true;245 }246 if (k < 0) { return false; }247 target = get_kv(ctx, k);248 return true;249 }250 251 static bool set(const gguf_context * ctx, const char * key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {252 return set(ctx, gguf_find_key(ctx, key), target, ovrd);253 }254 255 static bool set(const gguf_context * ctx, const std::string & key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {256 return set(ctx, key.c_str(), target, ovrd);257 }258 };259}260 261 template<typename T>262 typename std::enable_if<std::is_integral<T>::value, bool>::type263 llama_model_loader::get_arr_n(const std::string & key, T & result, bool required) {264 const int kid = gguf_find_key(meta.get(), key.c_str());265 266 if (kid < 0) {267 if (required) {268 throw std::runtime_error(format("key not found in model: %s", key.c_str()));269 }270 return false;271 }272 273 struct GGUFMeta::ArrayInfo arr_info =274 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta.get(), kid);275 276 277 result = arr_info.length;278 return true;279 }280 281 template<typename T>282 typename std::enable_if<std::is_integral<T>::value, bool>::type283 llama_model_loader::get_arr_n(enum llm_kv kid, T & result, bool required) {284 return get_arr_n(llm_kv(kid), result, required);285 }286 287 template bool llama_model_loader::get_arr_n(enum llm_kv kid, uint32_t & result, bool required);288 289 template<typename T>290 bool llama_model_loader::get_arr(const std::string & key, std::vector<T> & result, bool required) {291 const int kid = gguf_find_key(meta.get(), key.c_str());292 293 if (kid < 0 || gguf_get_kv_type(meta.get(), kid) != GGUF_TYPE_ARRAY) {294 if (required) {295 throw std::runtime_error(format("array key not found in model: %s", key.c_str()));296 }297 return false;298 }299 300 struct GGUFMeta::ArrayInfo arr_info =301 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta.get(), kid);302 303 switch (arr_info.gt) {304 case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value)); break;305 case GGUF_TYPE_INT32: GGML_ASSERT(306 (std::is_same<T, int32_t>::value) ||307 (std::is_same<T, uint32_t>::value)); break;308 default:309 throw std::runtime_error(format("%s is not a float32, int32 array", key.c_str()));310 }311 312 result.resize(arr_info.length);313 result.assign((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length);314 315 return true;316 }317 318 template<typename T, size_t N_MAX>319 bool llama_model_loader::get_arr(const std::string & key, std::array<T, N_MAX> & result, bool required) {320 const int kid = gguf_find_key(meta.get(), key.c_str());321 322 if (kid < 0 || gguf_get_kv_type(meta.get(), kid) != GGUF_TYPE_ARRAY) {323 if (required) {324 throw std::runtime_error(format("array key not found in model: %s", key.c_str()));325 }326 return false;327 }328 329 struct GGUFMeta::ArrayInfo arr_info =330 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta.get(), kid);331 332 switch (arr_info.gt) {333 case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value)); break;334 case GGUF_TYPE_INT32: GGML_ASSERT(335 (std::is_same<T, int32_t>::value) ||336 (std::is_same<T, uint32_t>::value)); break;337 default:338 throw std::runtime_error(format("%s is not a float32, int32 array", key.c_str()));339 }340 341 if (arr_info.length > N_MAX) {342 throw std::runtime_error(format("array length %u for key %s exceeds max %u", (uint32_t) arr_info.length, key.c_str(), (uint32_t) N_MAX));343 }344 345 std::copy((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length, result.begin());346 347 return true;348 }349 350 template<typename T>351 bool llama_model_loader::get_arr(enum llm_kv kid, T & result, bool required) {352 return get_arr(llm_kv(kid), result, required);353 }354 355 template<typename T>356 bool llama_model_loader::get_key(const std::string & key, T & result, bool required) {357 auto it = kv_overrides.find(key);358 359 const struct llama_model_kv_override * override =360 it != kv_overrides.end() ? &it->second : nullptr;361 362 const bool found = GGUFMeta::GKV<T>::set(meta.get(), key, result, override);363 364 if (required && !found) {365 throw std::runtime_error(format("key not found in model: %s", key.c_str()));366 }367 368 return found;369 }370 371 template<typename T>372 bool llama_model_loader::get_key(enum llm_kv kid, T & result, bool required) {373 return get_key(llm_kv(kid), result, required);374 }375 376 template bool llama_model_loader::get_key<bool> (enum llm_kv kid, bool & result, bool required);377 template bool llama_model_loader::get_key<float> (enum llm_kv kid, float & result, bool required);378 template bool llama_model_loader::get_key<uint32_t> (enum llm_kv kid, uint32_t & result, bool required);379 template bool llama_model_loader::get_key<std::string>(enum llm_kv kid, std::string & result, bool required);380 381 template<>382 bool llama_model_loader::get_key(enum llm_kv kid, enum llama_pooling_type & result, bool required) {383 uint32_t tmp;384 const bool found = get_key(kid, tmp, required);385 if (found) {386 result = (enum llama_pooling_type) tmp;387 } else {388 result = LLAMA_POOLING_TYPE_UNSPECIFIED;389 }390 return found;391 }392 393 // get array of n <= N_MAX elements, or a single element repeated n times394 template<typename T, size_t N_MAX>395 bool llama_model_loader::get_key_or_arr(const std::string & key, std::array<T, N_MAX> & result, uint32_t n, bool required) {396 const int kid = gguf_find_key(meta.get(), key.c_str());397 398 if (kid < 0) {399 if (required) {400 throw std::runtime_error(format("key not found in model: %s", key.c_str()));401 }402 return false;403 }404 405 if (n > N_MAX) {406 throw std::runtime_error(format("n > N_MAX: %u > %u for key %s", (uint32_t) n, (uint32_t) N_MAX, key.c_str()));407 }408 409 if (gguf_get_kv_type(meta.get(), kid) == GGUF_TYPE_ARRAY) {410 struct GGUFMeta::ArrayInfo arr_info =411 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(meta.get(), kid);412 413 if (n != arr_info.length) {414 throw std::runtime_error(format("key %s has wrong array length; expected %u, got %u", key.c_str(), n, (uint32_t) arr_info.length));415 }416 417 return get_arr(key, result, required);418 }419 420 T value;421 422 bool ok = get_key(key, value, required);423 if (!ok) {424 return false;425 }426 427 for (uint32_t i = 0; i < n; i++) {428 result[i] = value;429 }430 431 return true;432 }433 434 template<typename T>435 bool llama_model_loader::get_key_or_arr(enum llm_kv kid, T & result, uint32_t n, bool required) {436 return get_key_or_arr(llm_kv(kid), result, n, required);437 }438 439 // TODO: this is not very clever - figure out something better440 template bool llama_model_loader::get_key_or_arr<std::array<int, 4>>(enum llm_kv kid, std::array<int, 4> & result, uint32_t n, bool required);441 template bool llama_model_loader::get_key_or_arr<std::array<uint32_t, 512>>(enum llm_kv kid, std::array<uint32_t, 512> & result, uint32_t n, bool required);442 443llama_model_loader::llama_model_loader(444 const std::string & fname,445 std::vector<std::string> & splits,446 bool use_mmap,447 bool check_tensors,448 const struct llama_model_kv_override * param_overrides_p) {449 int trace = 0;450 if (getenv("LLAMA_TRACE")) {451 trace = atoi(getenv("LLAMA_TRACE"));452 }453 454 if (param_overrides_p != nullptr) {455 for (const struct llama_model_kv_override * p = param_overrides_p; p->key[0] != 0; p++) {456 kv_overrides.insert({std::string(p->key), *p});457 }458 }459 460 // Load the main GGUF461 struct ggml_context * ctx = NULL;462 struct gguf_init_params params = {463 /*.no_alloc = */ true,464 /*.ctx = */ &ctx,465 };466 467 meta.reset(gguf_init_from_file(fname.c_str(), params));468 if (!meta) {469 throw std::runtime_error(format("%s: failed to load model from %s\n", __func__, fname.c_str()));470 }471 472 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);473 llm_kv = LLM_KV(llm_arch_from_string(arch_name));474 475 files.emplace_back(new llama_file(fname.c_str(), "rb"));476 contexts.emplace_back(ctx);477 478 // Save tensors data offset of the main file.479 // For subsidiary files, `meta` tensor data offset must not be used,480 // so we build a unified tensors index for weights.481 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {482 std::string tensor_name = std::string(cur->name);483 // make sure there is no duplicated tensor names484 if (weights_map.find(tensor_name) != weights_map.end()) {485 throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));486 }487 n_elements += ggml_nelements(cur);488 n_bytes += ggml_nbytes(cur);489 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, meta.get(), cur));490 }491 uint16_t n_split = 0;492 get_key(llm_kv(LLM_KV_SPLIT_COUNT), n_split, false);493 494 // Load additional GGML contexts495 if (n_split > 1) {496 // make sure the main file is loaded first497 uint16_t idx = 0;498 const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO);499 get_key(kv_split_no, idx);500 if (idx != 0) {501 throw std::runtime_error(format("illegal split file idx: %d (file: %s), model must be loaded with the first split", idx, fname.c_str()));502 }503 504 // generate list of splits if needed505 if (splits.empty()) {506 splits = llama_get_list_splits(fname, idx, n_split);507 }508 509 // in case user give a custom list of splits, check if it matches the expected number510 if (n_split != (uint16_t)splits.size()) {511 throw std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", splits.size(), n_split));512 }513 514 if (trace > 0) {515 LLAMA_LOG_INFO("%s: loading additional %d GGUFs\n", __func__, n_split);516 }517 518 // load other splits519 for (idx = 1; idx < n_split; idx++) {520 const char * fname_split = splits[idx].c_str();521 522 struct gguf_init_params split_params = {523 /*.no_alloc = */ true,524 /*.ctx = */ &ctx,525 };526 gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) };527 if (!ctx_gguf) {528 throw std::runtime_error(format("%s: failed to load GGUF split from %s\n", __func__, fname_split));529 }530 531 // check idx532 {533 const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str());534 if (kid < 0) {535 throw std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split));536 }537 int idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid);538 if (idx_gguf != idx) {539 throw std::runtime_error(format("invalid split file idx: %d (file: %s), expected %d", idx_gguf, fname_split, idx));540 }541 }542 543 files.emplace_back(new llama_file(fname_split, "rb"));544 contexts.emplace_back(ctx);545 546 // Save tensors data offset info of the shard.547 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {548 std::string tensor_name = std::string(cur->name);549 // make sure there is no duplicated tensor names550 if (weights_map.find(tensor_name) != weights_map.end()) {551 throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));552 }553 n_elements += ggml_nelements(cur);554 n_bytes += ggml_nbytes(cur);555 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), idx, ctx_gguf.get(), cur));556 }557 }558 559 get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), n_tensors);560 561 // sanity check562 {563 const int n_tensors_loaded = (int) weights_map.size();564 if (n_tensors != n_tensors_loaded) {565 throw std::runtime_error(format("corrupted model: %d tensors expected but %d found", n_tensors, n_tensors_loaded));566 }567 }568 569 LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split - 1);570 }571 572 n_kv = gguf_get_n_kv(meta.get());573 n_tensors = weights_map.size();574 575 fver = (enum llama_fver) gguf_get_version(meta.get());576 577 LLAMA_LOG_INFO("%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n",578 __func__, n_kv, n_tensors, fname.c_str(), llama_file_version_name(fver));579 580 // determine file type based on the number of tensors for each quantization and print meta data581 // TODO: make optional582 {583 std::map<enum ggml_type, uint32_t> n_type;584 585 uint32_t n_type_max = 0;586 enum ggml_type type_max = GGML_TYPE_F32;587 588 for (const auto & it : weights_map) {589 const llama_tensor_weight & w = it.second;590 const ggml_tensor * tensor = w.tensor;591 592 enum ggml_type type = tensor->type;593 594 n_type[type]++;595 596 if (n_type_max < n_type[type]) {597 n_type_max = n_type[type];598 type_max = type;599 }600 601 if (trace > 0) {602 const uint16_t sid = w.idx;603 LLAMA_LOG_INFO("%s: - tensor split %2d: %32s %-8s [ %s ]\n", __func__, sid, ggml_get_name(tensor), ggml_type_name(type), llama_format_tensor_shape(tensor).c_str());604 }605 }606 607 switch (type_max) {608 case GGML_TYPE_F32: ftype = LLAMA_FTYPE_ALL_F32; break;609 case GGML_TYPE_F16: ftype = LLAMA_FTYPE_MOSTLY_F16; break;610 case GGML_TYPE_BF16: ftype = LLAMA_FTYPE_MOSTLY_BF16; break;611 case GGML_TYPE_Q4_0: ftype = LLAMA_FTYPE_MOSTLY_Q4_0; break;612 case GGML_TYPE_Q4_1: ftype = LLAMA_FTYPE_MOSTLY_Q4_1; break;613 case GGML_TYPE_Q5_0: ftype = LLAMA_FTYPE_MOSTLY_Q5_0; break;614 case GGML_TYPE_Q5_1: ftype = LLAMA_FTYPE_MOSTLY_Q5_1; break;615 case GGML_TYPE_Q8_0: ftype = LLAMA_FTYPE_MOSTLY_Q8_0; break;616 case GGML_TYPE_Q2_K: ftype = LLAMA_FTYPE_MOSTLY_Q2_K; break;617 case GGML_TYPE_Q3_K: ftype = LLAMA_FTYPE_MOSTLY_Q3_K_M; break;618 case GGML_TYPE_Q4_K: ftype = LLAMA_FTYPE_MOSTLY_Q4_K_M; break;619 case GGML_TYPE_Q5_K: ftype = LLAMA_FTYPE_MOSTLY_Q5_K_M; break;620 case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break;621 case GGML_TYPE_TQ1_0: ftype = LLAMA_FTYPE_MOSTLY_TQ1_0; break;622 case GGML_TYPE_TQ2_0: ftype = LLAMA_FTYPE_MOSTLY_TQ2_0; break;623 case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break;624 case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break;625 case GGML_TYPE_IQ2_S: ftype = LLAMA_FTYPE_MOSTLY_IQ2_S; break;626 case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break;627 case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break;628 case GGML_TYPE_IQ1_M: ftype = LLAMA_FTYPE_MOSTLY_IQ1_M; break;629 case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break;630 case GGML_TYPE_IQ4_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ4_XS; break;631 case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break;632 default:633 {634 LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));635 ftype = LLAMA_FTYPE_ALL_F32;636 } break;637 }638 639 // this is a way to mark that we have "guessed" the file type640 ftype = (llama_ftype) (ftype | LLAMA_FTYPE_GUESSED);641 642 {643 const int kid = gguf_find_key(meta.get(), "general.file_type"); // TODO: use LLM_KV644 if (kid >= 0) {645 ftype = (llama_ftype) gguf_get_val_u32(meta.get(), kid);646 }647 }648 649 LLAMA_LOG_INFO("%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", __func__);650 651 for (int i = 0; i < n_kv; i++) {652 const char * name = gguf_get_key(meta.get(), i);653 const enum gguf_type type = gguf_get_kv_type(meta.get(), i);654 const std::string type_name =655 type == GGUF_TYPE_ARRAY656 ? format("%s[%s,%zu]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(meta.get(), i)), gguf_get_arr_n(meta.get(), i))657 : gguf_type_name(type);658 659 std::string value = gguf_kv_to_str(meta.get(), i);660 const size_t MAX_VALUE_LEN = 40;661 if (value.size() > MAX_VALUE_LEN) {662 value = format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str());663 }664 replace_all(value, "\n", "\\n");665 666 LLAMA_LOG_INFO("%s: - kv %3d: %42s %-16s = %s\n", __func__, i, name, type_name.c_str(), value.c_str());667 }668 669 // print type counts670 for (auto & kv : n_type) {671 if (kv.second == 0) {672 continue;673 }674 675 LLAMA_LOG_INFO("%s: - type %4s: %4d tensors\n", __func__, ggml_type_name(kv.first), kv.second);676 }677 }678 679 if (!llama_mmap::SUPPORTED) {680 LLAMA_LOG_WARN("%s: mmap is not supported on this platform\n", __func__);681 use_mmap = false;682 }683 684 this->use_mmap = use_mmap;685 this->check_tensors = check_tensors;686}687 688std::string llama_model_loader::get_arch_name() const {689 return arch_name;690}691 692enum llm_arch llama_model_loader::get_arch() const {693 return llm_kv.arch;694}695 696const llama_model_loader::llama_tensor_weight * llama_model_loader::get_weight(const char * name) const {697 auto pos = weights_map.find(name);698 if (pos != weights_map.end()) {699 return &pos->second;700 }701 702 return nullptr;703}704 705const llama_model_loader::llama_tensor_weight & llama_model_loader::require_weight(const char * name) const {706 const llama_tensor_weight * weight = get_weight(name);707 if (!weight) {708 throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name));709 }710 return *weight;711}712 713struct ggml_tensor * llama_model_loader::get_tensor_meta(const char * name) const {714 const auto * weight = get_weight(name);715 if (!weight) {716 return nullptr;717 }718 return weight->tensor;719}720 721struct ggml_tensor * llama_model_loader::require_tensor_meta(const std::string & name) const {722 struct ggml_tensor * tensor = get_tensor_meta(name.c_str());723 if (!tensor) {724 throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));725 }726 return tensor;727}728 729const struct ggml_tensor * llama_model_loader::check_tensor_dims(const std::string & name, const std::vector<int64_t> & ne, bool required) const {730 const struct ggml_tensor * cur = get_tensor_meta(name.c_str());731 732 if (cur == NULL) {733 if (!required) {734 return NULL;735 }736 throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));737 }738 739 {740 bool is_ok = true;741 for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {742 if ((i < ne.size() && ne[i] != cur->ne[i]) || (i >= ne.size() && cur->ne[i] != 1)) {743 is_ok = false;744 break;745 }746 }747 if (!is_ok) {748 throw std::runtime_error(749 format("%s: tensor '%s' has wrong shape; expected %s, got %s",750 __func__, name.c_str(),751 llama_format_tensor_shape(ne).c_str(),752 llama_format_tensor_shape(cur).c_str()));753 }754 }755 756 return cur;757}758 759struct ggml_tensor * llama_model_loader::create_tensor(struct ggml_context * ctx, const std::string & name, const std::initializer_list<int64_t> & ne, int flags) {760 const struct ggml_tensor * cur = check_tensor_dims(name, ne, !(flags & TENSOR_NOT_REQUIRED));761 762 if (cur == NULL) {763 return NULL;764 }765 766 bool duplicated = flags & TENSOR_DUPLICATED;767 768 struct ggml_tensor * tensor = ggml_dup_tensor(ctx, cur);769 ggml_set_name(tensor, ggml_get_name(cur));770 771 if (duplicated) {772 size_data += ggml_nbytes(cur);773 } else {774 n_created++;775 }776 777 return tensor;778 779}780 781struct ggml_tensor * llama_model_loader::create_tensor_as_view(struct ggml_context * ctx, struct ggml_tensor * base, const std::string & name, const std::initializer_list<int64_t> & ne, size_t offset, bool required) {782 const struct ggml_tensor * cur = check_tensor_dims(name, ne, required);783 784 if (cur == NULL) {785 return NULL;786 }787 788 if (cur->type != base->type) {789 throw std::runtime_error(format("%s: tensor '%s' has wrong type; expected %s, got %s", __func__, name.c_str(), ggml_type_name(base->type), ggml_type_name(cur->type)));790 }791 792 std::array<int64_t, GGML_MAX_DIMS> dims;793 for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {794 dims[i] = i < ne.size() ? ne.begin()[i] : 1;795 }796 797 struct ggml_tensor * tensor = ggml_view_4d(ctx, base,798 dims[0], dims[1], dims[2], dims[3],799 cur->nb[1], cur->nb[2], cur->nb[3],800 offset);801 802 ggml_set_name(tensor, name.c_str());803 804 n_created++;805 806 return tensor;807}808 809void llama_model_loader::done_getting_tensors() const {810 if (n_created != n_tensors) {811 throw std::runtime_error(format("%s: wrong number of tensors; expected %d, got %d", __func__, n_tensors, n_created));812 }813}814 815void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps) {816 if (use_mmap) {817 mappings.reserve(files.size());818 mmaps_used.reserve(files.size());819 for (const auto & file : files) {820 auto * reg = ggml_backend_dev_backend_reg(ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU));821 auto * is_numa_fn = (decltype(ggml_is_numa) *) ggml_backend_reg_get_proc_address(reg, "ggml_backend_cpu_is_numa");822 std::unique_ptr<llama_mmap> mapping = std::make_unique<llama_mmap>(file.get(), prefetch ? -1 : 0, is_numa_fn());823 mmaps_used.emplace_back(mapping->size(), 0);824 if (mlock_mmaps) {825 std::unique_ptr<llama_mlock> mlock_mmap(new llama_mlock());826 mlock_mmap->init(mapping->addr());827 mlock_mmaps->emplace_back(std::move(mlock_mmap));828 }829 mappings.emplace_back(std::move(mapping));830 }831 }832 833 // compute the total size of all tensors for progress reporting834 for (const auto & it : weights_map) {835 size_data += ggml_nbytes(it.second.tensor);836 }837}838 839void llama_model_loader::get_mapping_range(size_t * first, size_t * last, void ** addr, int idx, ggml_context * ctx) const {840 GGML_ASSERT(!mappings.empty());841 const auto & mapping = mappings.at(idx);842 843 *first = mapping->size();844 *last = 0;845 *addr = mapping->addr();846 for (ggml_tensor * tensor = ggml_get_first_tensor(ctx); tensor; tensor = ggml_get_next_tensor(ctx, tensor)) {847 const auto * weight = get_weight(ggml_get_name(tensor));848 if (!weight || weight->idx != idx) {849 continue;850 }851 *first = std::min(*first, weight->offs);852 *last = std::max(*last, weight->offs + ggml_nbytes(tensor));853 }854}855 856void llama_model_loader::load_data_for(struct ggml_tensor * cur) const {857 const auto & w = require_weight(ggml_get_name(cur));858 859 if (use_mmap) {860 const auto & mapping = mappings.at(w.idx);861 if (cur->data == nullptr) {862 cur->data = (uint8_t *)mapping->addr() + w.offs;863 } else {864 memcpy(cur->data, (uint8_t *)mapping->addr() + w.offs, ggml_nbytes(cur));865 }866 } else {867 GGML_ASSERT(cur->data != nullptr);868 GGML_ASSERT(w.idx < files.size());869 const auto & file = files.at(w.idx);870 file->seek(w.offs, SEEK_SET);871 file->read_raw(cur->data, ggml_nbytes(cur));872 }873 874 if (check_tensors && !ggml_validate_row_data(cur->type, cur->data, ggml_nbytes(cur))) {875 throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));876 }877}878 879bool llama_model_loader::load_all_data(880 struct ggml_context * ctx,881 llama_buf_map & bufs,882 llama_mlocks * lmlocks,883 llama_progress_callback progress_callback,884 void * progress_callback_user_data) {885 GGML_ASSERT(size_data != 0 && "call init_mappings() first");886 887 std::vector<no_init<uint8_t>> read_buf;888 std::vector<std::future<std::pair<ggml_tensor *, bool>>> validation_result;889 890 // 4 staging buffers for async uploads, each sized 1MB seems to be a good default for single NVMe drives.891 // NVMe raid configurations might require more / larger buffers.892 constexpr size_t n_buffers = 4;893 constexpr size_t buffer_size = 1 * 1024 * 1024; // 1MB894 895 std::vector<ggml_backend_buffer_t> host_buffers;896 std::vector<ggml_backend_event_t> events;897 std::vector<void *> host_ptrs;898 size_t buffer_idx = 0; // buffer to use for async loads899 ggml_backend_t upload_backend = [&](const char * func) -> ggml_backend_t {900 if (use_mmap || check_tensors) {901 return nullptr;902 }903 // When not using mmaped io use async uploads from pinned memory to GPU memory.904 // First determine if the backend supports the necessary features for async uploads.905 auto * buf = bufs.count(0) ? bufs.at(0) : nullptr;906 if (!buf) {907 LLAMA_LOG_DEBUG("%s: no buffer found for async uploads\n", func);908 return nullptr;909 }910 911 auto * buft = ggml_backend_buffer_get_type(buf);912 auto * dev = ggml_backend_buft_get_device(buft);913 if (!dev) {914 LLAMA_LOG_DEBUG("%s: no device found for buffer type %s for async uploads\n", func,915 ggml_backend_buft_name(buft));916 return nullptr;917 }918 919 if (buft != ggml_backend_dev_buffer_type(dev)) {920 LLAMA_LOG_DEBUG("%s: buffer type %s is not the default buffer type for device %s for async uploads\n", func,921 ggml_backend_buft_name(buft), ggml_backend_dev_name(dev));922 return nullptr;923 }924 925 ggml_backend_dev_props props;926 ggml_backend_dev_get_props(dev, &props);927 if (!props.caps.async || !props.caps.host_buffer || !props.caps.events) {928 LLAMA_LOG_DEBUG("%s: device %s does not support async, host buffers or events\n", func,929 ggml_backend_dev_name(dev));930 return nullptr;931 }932 933 auto * host_buft = ggml_backend_dev_host_buffer_type(dev);934 if (!host_buft) {935 LLAMA_LOG_DEBUG("%s: no host buffer type found for device %s\n", func,936 ggml_backend_dev_name(dev));937 return nullptr;938 }939 940 // If the backend is supported, create pinned memory buffers and events for synchronisation.941 for (size_t idx = 0; idx < n_buffers; ++idx) {942 auto * buf = ggml_backend_buft_alloc_buffer(host_buft, buffer_size);943 if (!buf) {944 LLAMA_LOG_DEBUG("%s: failed to allocate host buffer for async uploads for device %s\n", func,945 ggml_backend_dev_name(dev));946 return nullptr;947 }948 949 host_buffers.emplace_back(buf);950 host_ptrs.emplace_back(ggml_backend_buffer_get_base(buf));951 952 auto * event = ggml_backend_event_new(dev);953 if (!event) {954 LLAMA_LOG_DEBUG("%s: failed to create event for async uploads for device %s\n", func,955 ggml_backend_dev_name(dev));956 return nullptr;957 }958 959 events.emplace_back(event);960 }961 962 ggml_backend_t backend = ggml_backend_dev_init(dev, nullptr);963 if (!backend) {964 LLAMA_LOG_DEBUG("%s: failed to initialize backend for device %s for async uploads\n", func,965 ggml_backend_dev_name(dev));966 return nullptr;967 }968 969 return backend;970 }(__func__);971 972 if (upload_backend) {973 LLAMA_LOG_DEBUG("%s: using async uploads for device %s, buffer type %s, backend %s\n", __func__,974 ggml_backend_dev_name(ggml_backend_get_device(upload_backend)),975 ggml_backend_buft_name(ggml_backend_buffer_get_type(bufs.at(0))),976 ggml_backend_name(upload_backend));977 }978 979 for (struct ggml_tensor * cur = ggml_get_first_tensor(ctx); cur != NULL; cur = ggml_get_next_tensor(ctx, cur)) {980 const auto * weight = get_weight(ggml_get_name(cur));981 if (weight == nullptr) {982 // this can happen with split experts models983 continue;984 }985 986 if (progress_callback) {987 if (!progress_callback((float) size_done / size_data, progress_callback_user_data)) {988 return false;989 }990 }991 992 size_t n_size = ggml_nbytes(cur);993 994 if (use_mmap) {995 const auto & mapping = mappings.at(weight->idx);996 ggml_backend_buffer_t buf_mmap = nullptr;997 if (bufs.count(weight->idx)) {998 buf_mmap = bufs.at(weight->idx);999 }1000 uint8_t * data = (uint8_t *) mapping->addr() + weight->offs;1001 1002 if (check_tensors) {1003 validation_result.emplace_back(std::async(std::launch::async, [cur, data, n_size] {1004 return std::make_pair(cur, ggml_validate_row_data(cur->type, data, n_size));1005 }));1006 }1007 1008 GGML_ASSERT(buf_mmap || cur->data); // either we have a buffer to allocate the tensor in, or it is already allocated1009 if (buf_mmap && cur->data == nullptr) {1010 ggml_backend_tensor_alloc(buf_mmap, cur, data);1011 if (lmlocks) {1012 const auto & lmlock = lmlocks->at(weight->idx);1013 lmlock->grow_to(weight->offs + n_size);1014 }1015 1016 auto & mmap_used = mmaps_used[weight->idx];1017 mmap_used.first = std::min(mmap_used.first, weight->offs);1018 mmap_used.second = std::max(mmap_used.second, weight->offs + n_size);1019 } else {1020 ggml_backend_tensor_set(cur, data, 0, n_size);1021 }1022 } else {1023 const auto & file = files.at(weight->idx);1024 if (ggml_backend_buffer_is_host(cur->buffer)) {1025 file->seek(weight->offs, SEEK_SET);1026 file->read_raw(cur->data, n_size);1027 if (check_tensors) {1028 validation_result.emplace_back(std::async(std::launch::async, [cur, n_size] {1029 return std::make_pair(cur, ggml_validate_row_data(cur->type, cur->data, n_size));1030 }));1031 }1032 } else {1033 // If upload_backend is valid load the tensor in chunks to pinned memory and upload the buffers asynchronously to the GPU.1034 if (upload_backend) {1035 file->seek(weight->offs, SEEK_SET);1036 1037 size_t bytes_read = 0;1038 1039 while (bytes_read < n_size) {1040 size_t read_iteration = std::min<size_t>(buffer_size, n_size - bytes_read);1041 1042 ggml_backend_event_synchronize(events[buffer_idx]);1043 file->read_raw(host_ptrs[buffer_idx], read_iteration);1044 ggml_backend_tensor_set_async(upload_backend, cur, host_ptrs[buffer_idx], bytes_read, read_iteration);1045 ggml_backend_event_record(events[buffer_idx], upload_backend);1046 1047 bytes_read += read_iteration;1048 ++buffer_idx;1049 buffer_idx %= n_buffers;1050 }1051 } else {1052 read_buf.resize(n_size);1053 file->seek(weight->offs, SEEK_SET);1054 file->read_raw(read_buf.data(), n_size);1055 ggml_backend_tensor_set(cur, read_buf.data(), 0, n_size);1056 if (check_tensors && !ggml_validate_row_data(cur->type, read_buf.data(), n_size)) {1057 throw std::runtime_error(format("tensor '%s' has invalid data", ggml_get_name(cur)));1058 }1059 }1060 }1061 }1062 1063 size_done += n_size;1064 }1065 1066 // free temporary resources used for async uploads1067 for (auto * event : events) {1068 ggml_backend_event_synchronize(event);1069 ggml_backend_event_free(event);1070 }1071 for (auto * buf : host_buffers) {1072 ggml_backend_buffer_free(buf);1073 }1074 ggml_backend_free(upload_backend);1075 1076 // check validation results1077 bool validation_failed = false;1078 for (auto & future : validation_result) {1079 auto result = future.get();1080 if (!result.second) {1081 LLAMA_LOG_ERROR("%s: tensor '%s' has invalid data\n", __func__, ggml_get_name(result.first));1082 validation_failed = true;1083 }1084 }1085 if (validation_failed) {1086 throw std::runtime_error("found tensors with invalid data");1087 }1088 1089 // check if this is the last call and do final cleanup1090 if (size_done >= size_data) {1091 // unmap offloaded tensors and metadata1092 if (use_mmap) {1093 for (uint32_t idx = 0; idx < mappings.size(); idx++) {1094 const auto & mmap_used = mmaps_used.at(idx);1095 auto & mapping = mappings.at(idx);1096 mapping->unmap_fragment(0, mmap_used.first);1097 if (mmap_used.second != 0) {1098 mapping->unmap_fragment(mmap_used.second, mapping->size());1099 }1100 }1101 }1102 if (progress_callback) {1103 // Even though the model is done loading, we still honor1104 // cancellation since we need to free allocations.1105 return progress_callback(1.0f, progress_callback_user_data);1106 }1107 }1108 1109 return true;1110}1111 1112std::string llama_model_loader::ftype_name() const {1113 return llama_model_ftype_name(ftype);1114}1115 1116void llama_model_loader::print_info() const {1117 LLAMA_LOG_INFO("%s: file format = %s\n", __func__, llama_file_version_name(fver));1118 LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_model_ftype_name(ftype).c_str());1119 if (n_bytes < GiB) {1120 LLAMA_LOG_INFO("%s: file size = %.2f MiB (%.2f BPW) \n", __func__, n_bytes/1024.0/1024.0, n_bytes*8.0/n_elements);1121 } else {1122 LLAMA_LOG_INFO("%s: file size = %.2f GiB (%.2f BPW) \n", __func__, n_bytes/1024.0/1024.0/1024.0, n_bytes*8.0/n_elements);1123 }1124}1125