Brunobkr/llama.cpp_AlgMor24_github
ΩFFFΣLLIa • llama.cpp • AlgMor24 ██████╗ ███████╗███████╗███████╗██╗ ██╗ ██╗ █████╗ ██╔═══██╗██╔════╝██╔════╝██╔════╝██║ ██║ ██║██╔══██╗ ██║ ██║█████╗ █████╗ █████╗ ██║ ██║ ██║███████║ ██║ ██║██╔══╝ ██╔══╝ ██╔══╝ ██║ ██║ ██║██╔══██║ ╚██████╔╝██║ ██║ ███████╗███████╗███████╗██║██║ ██║ ╚═════╝ ╚═╝ ╚═╝ ╚══════╝╚══════╝╚══════╝╚═╝╚═╝ ╚═╝ High-Performance LLM / VLM Inference & Autonomous Agentic Ecosystem… See the full description on the dataset page: https://huggingface.co/datasets/Brunobkr/llama.cpp_AlgMor24_github.
03.1k
1#include "llama-model-loader.h"2 3#include "ggml-alloc.h"4#include "ggml.h"5#include "gguf.h"6#include "llama-hparams.h"7#include "llama.h"8 9#include <algorithm>10#include <array>11#include <cinttypes>12#include <cstdint>13#include <cstring>14#include <future>15#include <regex>16 17static const size_t kiB = 1024;18static const size_t MiB = 1024*kiB;19static const size_t GiB = 1024*MiB;20 21const char * llama_file_version_name(llama_fver version) {22 switch (version) {23 case GGUF_FILE_VERSION_V1: return "GGUF V1 (support until nov 2023)";24 case GGUF_FILE_VERSION_V2: return "GGUF V2";25 case GGUF_FILE_VERSION_V3: return "GGUF V3 (latest)";26 }27 28 return "unknown";29}30 31#define LLAMA_FTYPE_PREFIX "(guessed) "32 33const char * llama_ftype_name(llama_ftype ftype) {34 static constexpr size_t guessed_prefix_len = sizeof(LLAMA_FTYPE_PREFIX) - 1;35 const char * name;36 switch ((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) {37 case LLAMA_FTYPE_ALL_F32: name = LLAMA_FTYPE_PREFIX "all F32"; break;38 case LLAMA_FTYPE_MOSTLY_F16: name = LLAMA_FTYPE_PREFIX "F16"; break;39 case LLAMA_FTYPE_MOSTLY_BF16: name = LLAMA_FTYPE_PREFIX "BF16"; break;40 case LLAMA_FTYPE_MOSTLY_Q1_0: name = LLAMA_FTYPE_PREFIX "Q1_0"; break;41 case LLAMA_FTYPE_MOSTLY_Q2_0: name = LLAMA_FTYPE_PREFIX "Q2_0"; break;42 case LLAMA_FTYPE_MOSTLY_Q4_0: name = LLAMA_FTYPE_PREFIX "Q4_0"; break;43 case LLAMA_FTYPE_MOSTLY_Q4_1: name = LLAMA_FTYPE_PREFIX "Q4_1"; break;44 case LLAMA_FTYPE_MOSTLY_Q5_0: name = LLAMA_FTYPE_PREFIX "Q5_0"; break;45 case LLAMA_FTYPE_MOSTLY_Q5_1: name = LLAMA_FTYPE_PREFIX "Q5_1"; break;46 case LLAMA_FTYPE_MOSTLY_Q8_0: name = LLAMA_FTYPE_PREFIX "Q8_0"; break;47 case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: name = LLAMA_FTYPE_PREFIX "MXFP4 MoE"; break;48 case LLAMA_FTYPE_MOSTLY_NVFP4: name = LLAMA_FTYPE_PREFIX "NVFP4"; break;49 case LLAMA_FTYPE_MOSTLY_Q2_K: name = LLAMA_FTYPE_PREFIX "Q2_K - Medium"; break;50 case LLAMA_FTYPE_MOSTLY_Q2_K_S: name = LLAMA_FTYPE_PREFIX "Q2_K - Small"; break;51 case LLAMA_FTYPE_MOSTLY_Q3_K_S: name = LLAMA_FTYPE_PREFIX "Q3_K - Small"; break;52 case LLAMA_FTYPE_MOSTLY_Q3_K_M: name = LLAMA_FTYPE_PREFIX "Q3_K - Medium"; break;53 case LLAMA_FTYPE_MOSTLY_Q3_K_L: name = LLAMA_FTYPE_PREFIX "Q3_K - Large"; break;54 case LLAMA_FTYPE_MOSTLY_Q4_K_S: name = LLAMA_FTYPE_PREFIX "Q4_K - Small"; break;55 case LLAMA_FTYPE_MOSTLY_Q4_K_M: name = LLAMA_FTYPE_PREFIX "Q4_K - Medium"; break;56 case LLAMA_FTYPE_MOSTLY_Q5_K_S: name = LLAMA_FTYPE_PREFIX "Q5_K - Small"; break;57 case LLAMA_FTYPE_MOSTLY_Q5_K_M: name = LLAMA_FTYPE_PREFIX "Q5_K - Medium"; break;58 case LLAMA_FTYPE_MOSTLY_Q6_K: name = LLAMA_FTYPE_PREFIX "Q6_K"; break;59 case LLAMA_FTYPE_MOSTLY_TQ1_0: name = LLAMA_FTYPE_PREFIX "TQ1_0 - 1.69 bpw ternary"; break;60 case LLAMA_FTYPE_MOSTLY_TQ2_0: name = LLAMA_FTYPE_PREFIX "TQ2_0 - 2.06 bpw ternary"; break;61 case LLAMA_FTYPE_MOSTLY_IQ2_XXS: name = LLAMA_FTYPE_PREFIX "IQ2_XXS - 2.0625 bpw"; break;62 case LLAMA_FTYPE_MOSTLY_IQ2_XS: name = LLAMA_FTYPE_PREFIX "IQ2_XS - 2.3125 bpw"; break;63 case LLAMA_FTYPE_MOSTLY_IQ2_S: name = LLAMA_FTYPE_PREFIX "IQ2_S - 2.5 bpw"; break;64 case LLAMA_FTYPE_MOSTLY_IQ2_M: name = LLAMA_FTYPE_PREFIX "IQ2_M - 2.7 bpw"; break;65 case LLAMA_FTYPE_MOSTLY_IQ3_XS: name = LLAMA_FTYPE_PREFIX "IQ3_XS - 3.3 bpw"; break;66 case LLAMA_FTYPE_MOSTLY_IQ3_XXS: name = LLAMA_FTYPE_PREFIX "IQ3_XXS - 3.0625 bpw"; break;67 case LLAMA_FTYPE_MOSTLY_IQ1_S: name = LLAMA_FTYPE_PREFIX "IQ1_S - 1.5625 bpw"; break;68 case LLAMA_FTYPE_MOSTLY_IQ1_M: name = LLAMA_FTYPE_PREFIX "IQ1_M - 1.75 bpw"; break;69 case LLAMA_FTYPE_MOSTLY_IQ4_NL: name = LLAMA_FTYPE_PREFIX "IQ4_NL - 4.5 bpw"; break;70 case LLAMA_FTYPE_MOSTLY_IQ4_XS: name = LLAMA_FTYPE_PREFIX "IQ4_XS - 4.25 bpw"; break;71 case LLAMA_FTYPE_MOSTLY_IQ3_S: name = LLAMA_FTYPE_PREFIX "IQ3_S - 3.4375 bpw"; break;72 case LLAMA_FTYPE_MOSTLY_IQ3_M: name = LLAMA_FTYPE_PREFIX "IQ3_S mix - 3.66 bpw"; break;73 default: name = LLAMA_FTYPE_PREFIX "unknown, may not work"; break;74 }75 return (ftype & LLAMA_FTYPE_GUESSED) ? name : name + guessed_prefix_len;76}77 78#undef LLAMA_FTYPE_PREFIX79 80// return a list of splits for a given path81// for example, given "<name>-00002-of-00004.gguf", returns list of all 4 splits82static std::vector<std::string> llama_get_list_splits(const std::string & path, const int idx, const int n_split) {83 std::vector<std::string> paths;84 std::string split_prefix;85 std::vector<char> buf(llama_path_max(), 0);86 87 {88 int ret = llama_split_prefix(buf.data(), buf.size(), path.c_str(), idx, n_split);89 if (!ret) {90 throw std::runtime_error(format("invalid split file name: %s", path.c_str()));91 }92 split_prefix = std::string(buf.data(), ret);93 }94 95 if (split_prefix.empty()) {96 throw std::runtime_error(format("invalid split file: %s", path.c_str()));97 }98 99 for (int idx = 0; idx < n_split; ++idx) {100 int ret = llama_split_path(buf.data(), buf.size(), split_prefix.c_str(), idx, n_split);101 paths.push_back(std::string(buf.data(), ret));102 }103 104 return paths;105}106 107namespace GGUFMeta {108 template <typename T, gguf_type gt_, T (*gfun)(const gguf_context *, const int64_t)>109 struct GKV_Base_Type {110 static constexpr gguf_type gt = gt_;111 112 static T getter(const gguf_context * ctx, const int kid) {113 return gfun(ctx, kid);114 }115 };116 117 template<typename T> struct GKV_Base;118 119 template<> struct GKV_Base<bool >: GKV_Base_Type<bool, GGUF_TYPE_BOOL, gguf_get_val_bool> {};120 template<> struct GKV_Base<uint8_t >: GKV_Base_Type<uint8_t, GGUF_TYPE_UINT8, gguf_get_val_u8 > {};121 template<> struct GKV_Base<uint16_t >: GKV_Base_Type<uint16_t, GGUF_TYPE_UINT16, gguf_get_val_u16 > {};122 template<> struct GKV_Base<uint32_t >: GKV_Base_Type<uint32_t, GGUF_TYPE_UINT32, gguf_get_val_u32 > {};123 template<> struct GKV_Base<uint64_t >: GKV_Base_Type<uint64_t, GGUF_TYPE_UINT64, gguf_get_val_u64 > {};124 template<> struct GKV_Base<int8_t >: GKV_Base_Type<int8_t, GGUF_TYPE_INT8, gguf_get_val_i8 > {};125 template<> struct GKV_Base<int16_t >: GKV_Base_Type<int16_t, GGUF_TYPE_INT16, gguf_get_val_i16 > {};126 template<> struct GKV_Base<int32_t >: GKV_Base_Type<int32_t, GGUF_TYPE_INT32, gguf_get_val_i32 > {};127 template<> struct GKV_Base<int64_t >: GKV_Base_Type<int64_t, GGUF_TYPE_INT64, gguf_get_val_i64 > {};128 template<> struct GKV_Base<float >: GKV_Base_Type<float, GGUF_TYPE_FLOAT32, gguf_get_val_f32 > {};129 template<> struct GKV_Base<double >: GKV_Base_Type<double, GGUF_TYPE_FLOAT64, gguf_get_val_f64 > {};130 template<> struct GKV_Base<const char *>: GKV_Base_Type<const char *, GGUF_TYPE_STRING, gguf_get_val_str > {};131 132 template<> struct GKV_Base<std::string> {133 static constexpr gguf_type gt = GGUF_TYPE_STRING;134 135 static std::string getter(const gguf_context * ctx, const int kid) {136 return gguf_get_val_str(ctx, kid);137 }138 };139 140 struct ArrayInfo {141 const gguf_type gt;142 const size_t length;143 const void * data;144 };145 146 template<> struct GKV_Base<ArrayInfo> {147 public:148 static constexpr gguf_type gt = GGUF_TYPE_ARRAY;149 static ArrayInfo getter(const gguf_context *ctx, const int k) {150 const enum gguf_type arr_type = gguf_get_arr_type(ctx, k);151 return ArrayInfo {152 arr_type,153 gguf_get_arr_n(ctx, k),154 arr_type == GGUF_TYPE_STRING ? nullptr : gguf_get_arr_data(ctx, k),155 };156 }157 };158 159 template<typename T>160 class GKV : public GKV_Base<T> {161 GKV() = delete;162 163 public:164 static T get_kv(const gguf_context * ctx, const int k) {165 const enum gguf_type kt = gguf_get_kv_type(ctx, k);166 167 if (kt != GKV::gt) {168 throw std::runtime_error(format("key %s has wrong type %s but expected type %s",169 gguf_get_key(ctx, k), gguf_type_name(kt), gguf_type_name(GKV::gt)));170 }171 return GKV::getter(ctx, k);172 }173 174 static const char * override_type_to_str(const llama_model_kv_override_type ty) {175 switch (ty) {176 case LLAMA_KV_OVERRIDE_TYPE_BOOL: return "bool";177 case LLAMA_KV_OVERRIDE_TYPE_INT: return "int";178 case LLAMA_KV_OVERRIDE_TYPE_FLOAT: return "float";179 case LLAMA_KV_OVERRIDE_TYPE_STR: return "str";180 }181 return "unknown";182 }183 184 static bool validate_override(const llama_model_kv_override_type expected_type, const struct llama_model_kv_override * ovrd) {185 if (!ovrd) { return false; }186 if (ovrd->tag == expected_type) {187 LLAMA_LOG_INFO("%s: Using metadata override (%5s) '%s' = ",188 __func__, override_type_to_str(ovrd->tag), ovrd->key);189 switch (ovrd->tag) {190 case LLAMA_KV_OVERRIDE_TYPE_BOOL: {191 LLAMA_LOG_INFO("%s\n", ovrd->val_bool ? "true" : "false");192 } break;193 case LLAMA_KV_OVERRIDE_TYPE_INT: {194 LLAMA_LOG_INFO("%" PRId64 "\n", ovrd->val_i64);195 } break;196 case LLAMA_KV_OVERRIDE_TYPE_FLOAT: {197 LLAMA_LOG_INFO("%.6f\n", ovrd->val_f64);198 } break;199 case LLAMA_KV_OVERRIDE_TYPE_STR: {200 LLAMA_LOG_INFO("%s\n", ovrd->val_str);201 } break;202 default:203 // Shouldn't be possible to end up here, but just in case...204 throw std::runtime_error(205 format("Unsupported attempt to override %s type for metadata key %s\n",206 override_type_to_str(ovrd->tag), ovrd->key));207 }208 return true;209 }210 LLAMA_LOG_WARN("%s: Warning: Bad metadata override type for key '%s', expected %s but got %s\n",211 __func__, ovrd->key, override_type_to_str(expected_type), override_type_to_str(ovrd->tag));212 return false;213 }214 215 template<typename OT>216 static typename std::enable_if<std::is_same<OT, bool>::value, bool>::type217 try_override(OT & target, const struct llama_model_kv_override * ovrd) {218 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_BOOL, ovrd)) {219 target = ovrd->val_bool;220 return true;221 }222 return false;223 }224 225 template<typename OT>226 static typename std::enable_if<!std::is_same<OT, bool>::value && std::is_integral<OT>::value, bool>::type227 try_override(OT & target, const struct llama_model_kv_override * ovrd) {228 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_INT, ovrd)) {229 target = ovrd->val_i64;230 return true;231 }232 return false;233 }234 235 template<typename OT>236 static typename std::enable_if<std::is_floating_point<OT>::value, bool>::type237 try_override(T & target, const struct llama_model_kv_override * ovrd) {238 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_FLOAT, ovrd)) {239 target = ovrd->val_f64;240 return true;241 }242 return false;243 }244 245 template<typename OT>246 static typename std::enable_if<std::is_same<OT, std::string>::value, bool>::type247 try_override(T & target, const struct llama_model_kv_override * ovrd) {248 if (validate_override(LLAMA_KV_OVERRIDE_TYPE_STR, ovrd)) {249 target = ovrd->val_str;250 return true;251 }252 return false;253 }254 255 static bool set(const gguf_context * ctx, const int k, T & target, const struct llama_model_kv_override * ovrd = nullptr) {256 if (try_override<T>(target, ovrd)) {257 return true;258 }259 if (k < 0) { return false; }260 target = get_kv(ctx, k);261 return true;262 }263 264 static bool set(const gguf_context * ctx, const char * key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {265 return set(ctx, gguf_find_key(ctx, key), target, ovrd);266 }267 268 static bool set(const gguf_context * ctx, const std::string & key, T & target, const struct llama_model_kv_override * ovrd = nullptr) {269 return set(ctx, key.c_str(), target, ovrd);270 }271 };272}273 274 template<typename T>275 typename std::enable_if<std::is_integral<T>::value, bool>::type276 llama_model_loader::get_arr_n(const std::string & key, T & result, bool required) {277 const int kid = gguf_find_key(metadata, key.c_str());278 279 if (kid < 0) {280 if (required) {281 throw std::runtime_error(format("key not found in model: %s", key.c_str()));282 }283 return false;284 }285 286 struct GGUFMeta::ArrayInfo arr_info =287 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(metadata, kid);288 289 290 result = arr_info.length;291 return true;292 }293 294 template<typename T>295 typename std::enable_if<std::is_integral<T>::value, bool>::type296 llama_model_loader::get_arr_n(enum llm_kv kid, T & result, bool required) {297 return get_arr_n(llm_kv(kid), result, required);298 }299 300 template bool llama_model_loader::get_arr_n(enum llm_kv kid, uint32_t & result, bool required);301 template std::enable_if<std::is_integral<uint32_t>::value, bool>::type302 llama_model_loader::get_arr_n<uint32_t>(const std::string & key, uint32_t & result, bool required);303 304 template<typename T>305 bool llama_model_loader::get_arr(const std::string & key, std::vector<T> & result, bool required) {306 const gguf_context * ctx = metadata;307 const int kid = gguf_find_key(ctx, key.c_str());308 309 if (kid < 0 || gguf_get_kv_type(ctx, kid) != GGUF_TYPE_ARRAY) {310 if (required) {311 throw std::runtime_error(format("array key not found in model: %s", key.c_str()));312 }313 return false;314 }315 316 struct GGUFMeta::ArrayInfo arr_info =317 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx, kid);318 319 switch (arr_info.gt) {320 case GGUF_TYPE_UINT32:321 case GGUF_TYPE_INT32: GGML_ASSERT((std::is_same<T, int32_t>::value) ||322 (std::is_same<T, uint32_t>::value)); break;323 case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value)); break;324 case GGUF_TYPE_STRING: GGML_ASSERT((std::is_same<T, std::string>::value)); break;325 default:326 throw std::runtime_error(format("%s is not a string/float32/uint32/int32 array", key.c_str()));327 }328 329 if constexpr (std::is_same<T, std::string>::value) {330 const size_t n_items = gguf_get_arr_n(ctx, kid);331 result.clear();332 333 for (size_t i = 0; i < n_items; i++) {334 const T value = gguf_get_arr_str(ctx, kid, i);335 result.emplace_back(value);336 }337 } else {338 result.resize(arr_info.length);339 result.assign((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length);340 }341 342 return true;343 }344 345 template<typename T, size_t N_MAX>346 bool llama_model_loader::get_arr(const std::string & key, std::array<T, N_MAX> & result, bool required) {347 const gguf_context * ctx = metadata;348 const int kid = gguf_find_key(ctx, key.c_str());349 350 if (kid < 0 || gguf_get_kv_type(ctx, kid) != GGUF_TYPE_ARRAY) {351 if (required) {352 throw std::runtime_error(format("array key not found in model: %s", key.c_str()));353 }354 return false;355 }356 357 struct GGUFMeta::ArrayInfo arr_info =358 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(ctx, kid);359 360 switch (arr_info.gt) {361 case GGUF_TYPE_BOOL:362 case GGUF_TYPE_UINT32:363 case GGUF_TYPE_INT32: GGML_ASSERT((std::is_same<T, int32_t>::value) ||364 (std::is_same<T, uint32_t>::value)); break;365 case GGUF_TYPE_FLOAT32: GGML_ASSERT((std::is_same<T, float>::value)); break;366 case GGUF_TYPE_STRING: GGML_ASSERT((std::is_same<T, std::string>::value)); break;367 default:368 throw std::runtime_error(format("%s is not a string/float32/uint32/int32 array", key.c_str()));369 }370 371 if (arr_info.length > N_MAX) {372 throw std::runtime_error(format("array length %u for key %s exceeds max %u", (uint32_t) arr_info.length, key.c_str(), (uint32_t) N_MAX));373 }374 375 if constexpr (std::is_same<T, std::string>::value) {376 const size_t n_items = gguf_get_arr_n(ctx, kid);377 378 for (size_t i = 0; i < n_items; i++) {379 const T value = gguf_get_arr_str(ctx, kid, i);380 result[i] = value;381 }382 } else {383 if (arr_info.gt == GGUF_TYPE_BOOL) {384 const int8_t * values = (const int8_t *) arr_info.data;385 std::transform(values, values + arr_info.length, result.begin(), [](int8_t x) {386 return static_cast<T>(x != 0);387 });388 } else {389 std::copy((const T*)arr_info.data, (const T *)arr_info.data + arr_info.length, result.begin());390 }391 }392 393 return true;394 }395 396 template<typename T>397 bool llama_model_loader::get_arr(enum llm_kv kid, T & result, bool required) {398 return get_arr(llm_kv(kid), result, required);399 }400 401 template bool llama_model_loader::get_arr<std::vector<std::string>>(enum llm_kv kid, std::vector<std::string> & result, bool required);402 template bool llama_model_loader::get_arr<std::array<int32_t, 512>>(enum llm_kv kid, std::array<int32_t, 512> & result, bool required);403 template bool llama_model_loader::get_arr<std::vector<int32_t>>(enum llm_kv kid, std::vector<int32_t> & result, bool required);404 template bool llama_model_loader::get_arr<std::array<uint32_t, LLAMA_MAX_LAYERS>>(enum llm_kv kid, std::array<uint32_t, LLAMA_MAX_LAYERS> & result, bool required);405 406 template<typename T>407 bool llama_model_loader::get_key(const std::string & key, T & result, bool required) {408 auto it = kv_overrides.find(key);409 410 const struct llama_model_kv_override * override =411 it != kv_overrides.end() ? &it->second : nullptr;412 413 const bool found = GGUFMeta::GKV<T>::set(metadata, key, result, override);414 415 if (required && !found) {416 throw std::runtime_error(format("key not found in model: %s", key.c_str()));417 }418 419 return found;420 }421 422 template<typename T>423 bool llama_model_loader::get_key(enum llm_kv kid, T & result, bool required) {424 return get_key(llm_kv(kid), result, required);425 }426 427 template bool llama_model_loader::get_key<bool> (enum llm_kv kid, bool & result, bool required);428 template bool llama_model_loader::get_key<float> (enum llm_kv kid, float & result, bool required);429 template bool llama_model_loader::get_key<uint32_t> (enum llm_kv kid, uint32_t & result, bool required);430 template bool llama_model_loader::get_key<std::string>(enum llm_kv kid, std::string & result, bool required);431 432 template<>433 bool llama_model_loader::get_key(enum llm_kv kid, enum llama_pooling_type & result, bool required) {434 uint32_t tmp;435 const bool found = get_key(kid, tmp, required);436 if (found) {437 result = (enum llama_pooling_type) tmp;438 } else {439 result = LLAMA_POOLING_TYPE_UNSPECIFIED;440 }441 return found;442 }443 444 // get array of n <= N_MAX elements, or a single element repeated n times445 template<typename T, size_t N_MAX>446 bool llama_model_loader::get_key_or_arr(const std::string & key, std::array<T, N_MAX> & result, uint32_t n, bool required) {447 const int kid = gguf_find_key(metadata, key.c_str());448 449 if (kid < 0) {450 if (required) {451 throw std::runtime_error(format("key not found in model: %s", key.c_str()));452 }453 return false;454 }455 456 if (n > N_MAX) {457 throw std::runtime_error(format("n > N_MAX: %u > %u for key %s", n, (uint32_t) N_MAX, key.c_str()));458 }459 460 if (gguf_get_kv_type(metadata, kid) == GGUF_TYPE_ARRAY) {461 struct GGUFMeta::ArrayInfo arr_info =462 GGUFMeta::GKV<GGUFMeta::ArrayInfo>::get_kv(metadata, kid);463 464 if (n != arr_info.length) {465 throw std::runtime_error(format("key %s has wrong array length; expected %u, got %u", key.c_str(), n, (uint32_t) arr_info.length));466 }467 468 return get_arr(key, result, required);469 }470 471 T value;472 473 bool ok = get_key(key, value, required);474 if (!ok) {475 return false;476 }477 478 for (uint32_t i = 0; i < n; i++) {479 result[i] = value;480 }481 482 return true;483 }484 485 template<typename T>486 bool llama_model_loader::get_key_or_arr(enum llm_kv kid, T & result, uint32_t n, bool required) {487 return get_key_or_arr(llm_kv(kid), result, n, required);488 }489 490 bool llama_model_loader::get_key_or_arr(enum llm_kv kid, uint32_t & result, bool required) {491 const std::string key = llm_kv(kid);492 493 const int id = gguf_find_key(metadata, key.c_str());494 495 if (id < 0) {496 if (required) {497 throw std::runtime_error(format("key not found in model: %s", key.c_str()));498 }499 return false;500 }501 502 // throw and error if type is an array503 if (gguf_get_kv_type(metadata, id) == GGUF_TYPE_ARRAY) {504 if (required) {505 throw std::runtime_error(format("expected scalar, found array for key: %s", key.c_str()));506 }507 return false;508 }509 510 return get_key(key, result, required);511 }512 513 // TODO: this is not very clever - figure out something better514 template bool llama_model_loader::get_key_or_arr<std::array<int, 4>> (enum llm_kv kid, std::array<int, 4> & result, uint32_t n, bool required);515 template bool llama_model_loader::get_key_or_arr<std::array<uint32_t, 512>>(enum llm_kv kid, std::array<uint32_t, 512> & result, uint32_t n, bool required);516 template bool llama_model_loader::get_key_or_arr<std::array<float, 512>>(enum llm_kv kid, std::array<float, 512> & result, uint32_t n, bool required);517 518 519llama_model_loader::llama_model_loader(520 struct gguf_context * meta,521 llama_model_set_tensor_data_t set_tensor_data,522 void * set_tensor_data_ud,523 const std::string & fname,524 std::vector<std::string> & splits,525 FILE * file,526 llama_load_mode load_mode,527 bool check_tensors,528 bool no_alloc,529 bool load_mtp,530 const llama_model_kv_override * param_overrides_p,531 const llama_model_tensor_buft_override * param_tensor_buft_overrides_p)532 : metadata(meta), set_tensor_data(set_tensor_data), set_tensor_data_ud(set_tensor_data_ud) {533 int trace = 0;534 if (getenv("LLAMA_TRACE")) {535 trace = atoi(getenv("LLAMA_TRACE"));536 }537 538 if (param_overrides_p != nullptr) {539 for (const struct llama_model_kv_override * p = param_overrides_p; p->key[0] != 0; p++) {540 kv_overrides.insert({std::string(p->key), *p});541 }542 }543 544 tensor_buft_overrides = param_tensor_buft_overrides_p;545 546 this->use_mmap = load_mode == LLAMA_LOAD_MODE_MMAP || load_mode == LLAMA_LOAD_MODE_MMAP_MLOCK;547 this->use_direct_io = load_mode == LLAMA_LOAD_MODE_DIRECT_IO;548 549 if (!fname.empty()) {550 // Load the main GGUF551 struct ggml_context * ctx = NULL;552 struct gguf_init_params params = {553 /*.no_alloc = */ true,554 /*.ctx = */ &ctx,555 };556 557 metadata_ptr.reset(gguf_init_from_file(fname.c_str(), params));558 metadata = metadata_ptr.get();559 if (metadata == nullptr) {560 throw std::runtime_error(format("%s: failed to load model from %s", __func__, fname.c_str()));561 }562 563 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);564 llm_kv = LLM_KV(llm_arch_from_string(arch_name));565 566 files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io));567 contexts.emplace_back(ctx);568 569 // Save tensors data offset of the main file.570 // For subsidiary files, `meta` tensor data offset must not be used,571 // so we build a unified tensors index for weights.572 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {573 std::string tensor_name = std::string(cur->name);574 // make sure there is no duplicated tensor names575 if (weights_map.find(tensor_name) != weights_map.end()) {576 throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));577 }578 n_elements += ggml_nelements(cur);579 n_bytes += ggml_nbytes(cur);580 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur));581 }582 uint16_t n_split = 0;583 get_key(llm_kv(LLM_KV_SPLIT_COUNT), n_split, false);584 585 // Load additional GGML contexts586 if (n_split > 1) {587 // make sure the main file is loaded first588 uint16_t idx = 0;589 const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO);590 get_key(kv_split_no, idx);591 if (idx != 0) {592 throw std::runtime_error(format("illegal split file idx: %d (file: %s), model must be loaded with the first split", idx, fname.c_str()));593 }594 595 // generate list of splits if needed596 if (splits.empty()) {597 splits = llama_get_list_splits(fname, idx, n_split);598 }599 600 // in case user give a custom list of splits, check if it matches the expected number601 if (n_split != (uint16_t)splits.size()) {602 throw std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", splits.size(), n_split));603 }604 605 if (trace > 0) {606 LLAMA_LOG_INFO("%s: loading additional %d GGUFs\n", __func__, n_split);607 }608 609 // load other splits610 for (idx = 1; idx < n_split; idx++) {611 const char * fname_split = splits[idx].c_str();612 613 struct gguf_init_params split_params = {614 /*.no_alloc = */ true,615 /*.ctx = */ &ctx,616 };617 gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) };618 if (!ctx_gguf) {619 throw std::runtime_error(format("%s: failed to load GGUF split from %s", __func__, fname_split));620 }621 622 // check idx623 {624 const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str());625 if (kid < 0) {626 throw std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split));627 }628 int idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid);629 if (idx_gguf != idx) {630 throw std::runtime_error(format("invalid split file idx: %d (file: %s), expected %d", idx_gguf, fname_split, idx));631 }632 }633 634 files.emplace_back(new llama_file(fname_split, "rb", use_direct_io));635 contexts.emplace_back(ctx);636 637 // Save tensors data offset info of the shard.638 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {639 std::string tensor_name = std::string(cur->name);640 // make sure there is no duplicated tensor names641 if (weights_map.find(tensor_name) != weights_map.end()) {642 throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));643 }644 n_elements += ggml_nelements(cur);645 n_bytes += ggml_nbytes(cur);646 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), idx, ctx_gguf.get(), cur));647 }648 }649 650 get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), n_tensors);651 652 // sanity check653 {654 const int n_tensors_loaded = (int) weights_map.size();655 if (n_tensors != n_tensors_loaded) {656 throw std::runtime_error(format("corrupted model: %d tensors expected but %d found", n_tensors, n_tensors_loaded));657 }658 }659 660 LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split - 1);661 }662 } else if (file != nullptr) {663 struct ggml_context * ctx = NULL;664 struct gguf_init_params params = {665 /*.no_alloc = */ true,666 /*.ctx = */ &ctx,667 };668 669 metadata_ptr.reset(gguf_init_from_file_ptr(file, params));670 metadata = metadata_ptr.get();671 if (metadata == nullptr) {672 throw std::runtime_error(format("%s: failed to load model from file pointer", __func__));673 }674 675 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);676 llm_kv = LLM_KV(llm_arch_from_string(arch_name));677 678 files.emplace_back(new llama_file(file));679 contexts.emplace_back(ctx);680 681 // Save tensors data offset info of the main file.682 for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) {683 std::string tensor_name = std::string(cur->name);684 // make sure there is no duplicated tensor names685 if (weights_map.find(tensor_name) != weights_map.end()) {686 throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur)));687 }688 n_elements += ggml_nelements(cur);689 n_bytes += ggml_nbytes(cur);690 weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur));691 }692 } else {693 get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false);694 llm_kv = LLM_KV(llm_arch_from_string(arch_name));695 }696 697 n_kv = gguf_get_n_kv(metadata);698 n_tensors = weights_map.size();699 700 fver = (enum llama_fver) gguf_get_version(metadata);701 702 LLAMA_LOG_INFO("%s: loaded meta data with %d key-value pairs and %d tensors from %s (version %s)\n",703 __func__, n_kv, n_tensors, fname.empty() ? "(file*)" : fname.c_str(), llama_file_version_name(fver));704 705 // determine file type based on the number of tensors for each quantization and print meta data706 // TODO: make optional707 {708 std::map<enum ggml_type, uint32_t> n_type;709 710 uint32_t n_type_max = 0;711 enum ggml_type type_max = GGML_TYPE_F32;712 713 for (const auto & it : weights_map) {714 const llama_tensor_weight & w = it.second;715 const ggml_tensor * tensor = w.tensor;716 717 enum ggml_type type = tensor->type;718 719 n_type[type]++;720 721 if (n_type_max < n_type[type]) {722 n_type_max = n_type[type];723 type_max = type;724 }725 726 if (trace > 0) {727 const uint16_t sid = w.idx;728 LLAMA_LOG_INFO("%s: - tensor split %2d: %32s %-8s [ %s ] %8.2f MiB\n", __func__,729 sid, ggml_get_name(tensor), ggml_type_name(type), llama_format_tensor_shape(tensor).c_str(),730 ggml_nbytes(tensor)/1024.0f/1024.0f);731 }732 }733 734 switch (type_max) {735 case GGML_TYPE_F32: ftype = LLAMA_FTYPE_ALL_F32; break;736 case GGML_TYPE_F16: ftype = LLAMA_FTYPE_MOSTLY_F16; break;737 case GGML_TYPE_BF16: ftype = LLAMA_FTYPE_MOSTLY_BF16; break;738 case GGML_TYPE_Q4_0: ftype = LLAMA_FTYPE_MOSTLY_Q4_0; break;739 case GGML_TYPE_Q4_1: ftype = LLAMA_FTYPE_MOSTLY_Q4_1; break;740 case GGML_TYPE_Q5_0: ftype = LLAMA_FTYPE_MOSTLY_Q5_0; break;741 case GGML_TYPE_Q5_1: ftype = LLAMA_FTYPE_MOSTLY_Q5_1; break;742 case GGML_TYPE_Q8_0: ftype = LLAMA_FTYPE_MOSTLY_Q8_0; break;743 case GGML_TYPE_Q2_K: ftype = LLAMA_FTYPE_MOSTLY_Q2_K; break;744 case GGML_TYPE_Q3_K: ftype = LLAMA_FTYPE_MOSTLY_Q3_K_M; break;745 case GGML_TYPE_Q4_K: ftype = LLAMA_FTYPE_MOSTLY_Q4_K_M; break;746 case GGML_TYPE_Q5_K: ftype = LLAMA_FTYPE_MOSTLY_Q5_K_M; break;747 case GGML_TYPE_Q6_K: ftype = LLAMA_FTYPE_MOSTLY_Q6_K; break;748 case GGML_TYPE_TQ1_0: ftype = LLAMA_FTYPE_MOSTLY_TQ1_0; break;749 case GGML_TYPE_TQ2_0: ftype = LLAMA_FTYPE_MOSTLY_TQ2_0; break;750 case GGML_TYPE_IQ2_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XXS; break;751 case GGML_TYPE_IQ2_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ2_XS; break;752 case GGML_TYPE_IQ2_S: ftype = LLAMA_FTYPE_MOSTLY_IQ2_S; break;753 case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break;754 case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break;755 case GGML_TYPE_IQ1_M: ftype = LLAMA_FTYPE_MOSTLY_IQ1_M; break;756 case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break;757 case GGML_TYPE_IQ4_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ4_XS; break;758 case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break;759 case GGML_TYPE_NVFP4: ftype = LLAMA_FTYPE_MOSTLY_NVFP4; break;760 case GGML_TYPE_Q1_0: ftype = LLAMA_FTYPE_MOSTLY_Q1_0; break;761 case GGML_TYPE_Q2_0: ftype = LLAMA_FTYPE_MOSTLY_Q2_0; break;762 default:763 {764 LLAMA_LOG_WARN("%s: unknown type %s\n", __func__, ggml_type_name(type_max));765 ftype = LLAMA_FTYPE_ALL_F32;766 } break;767 }768 769 // this is a way to mark that we have "guessed" the file type770 ftype = (llama_ftype) (ftype | LLAMA_FTYPE_GUESSED);771 772 {773 uint32_t ftype_val = 0;774 if (get_key(LLM_KV_GENERAL_FILE_TYPE, ftype_val, false)) {775 ftype = (llama_ftype) ftype_val;776 }777 }778 779 LLAMA_LOG_INFO("%s: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n", __func__);780 781 for (int i = 0; i < n_kv; i++) {782 const char * name = gguf_get_key(metadata, i);783 const enum gguf_type type = gguf_get_kv_type(metadata, i);784 const std::string type_name =785 type == GGUF_TYPE_ARRAY786 ? format("%s[%s,%zu]", gguf_type_name(type), gguf_type_name(gguf_get_arr_type(metadata, i)), gguf_get_arr_n(metadata, i))787 : gguf_type_name(type);788 789 std::string value = gguf_kv_to_str(metadata, i);790 const size_t MAX_VALUE_LEN = 40;791 if (value.size() > MAX_VALUE_LEN) {792 value = format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str());793 }794 replace_all(value, "\n", "\\n");795 796 LLAMA_LOG_INFO("%s: - kv %3d: %42s %-16s = %s\n", __func__, i, name, type_name.c_str(), value.c_str());797 }798 799 // print type counts800 for (auto & kv : n_type) {801 if (kv.second == 0) {802 continue;803 }804 805 LLAMA_LOG_INFO("%s: - type %4s: %4d tensors\n", __func__, ggml_type_name(kv.first), kv.second);806 }807 }808 809 if (this->use_mmap && !llama_mmap::SUPPORTED) {810 LLAMA_LOG_WARN("%s: mmap is not supported on this platform\n", __func__);811 this->use_mmap = false;812 }813 814 this->check_tensors = check_tensors;815 this->no_alloc = no_alloc;816 this->load_mtp = load_mtp;817}818 819std::string llama_model_loader::get_arch_name() const {820 return arch_name;821}822 823enum llm_arch llama_model_loader::get_arch() const {824 return llm_kv.arch;825}826 827const llama_model_loader::llama_tensor_weight * llama_model_loader::get_weight(const char * name) const {828 auto pos = weights_map.find(name);829 if (pos != weights_map.end()) {830 return &pos->second;831 }832 833 return nullptr;834}835 836const llama_model_loader::llama_tensor_weight & llama_model_loader::require_weight(const char * name) const {837 const llama_tensor_weight * weight = get_weight(name);838 if (!weight) {839 throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name));840 }841 return *weight;842}843 844struct ggml_tensor * llama_model_loader::get_tensor_meta(const char * name) const {845 const auto * weight = get_weight(name);846 if (!weight) {847 return nullptr;848 }849 return weight->tensor;850}851 852struct ggml_tensor * llama_model_loader::require_tensor_meta(const std::string & name) const {853 struct ggml_tensor * tensor = get_tensor_meta(name.c_str());854 if (!tensor) {855 throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));856 }857 return tensor;858}859 860const struct ggml_tensor * llama_model_loader::check_tensor_dims(861 const std::string & name,862 const std::vector<int64_t> & ne,863 bool required,864 bool allow_reshape) const {865 const struct ggml_tensor * cur = get_tensor_meta(name.c_str());866 867 if (cur == NULL) {868 if (!required) {869 return NULL;870 }871 throw std::runtime_error(format("%s: tensor '%s' not found", __func__, name.c_str()));872 }873 874 bool is_ok = true;875 876 if (allow_reshape) {877 // check total number of elements only878 const int64_t ncur = ggml_nelements(cur);879 int64_t nexp = 1;880 for (size_t i = 0; i < ne.size(); ++i) {881 nexp *= ne[i];882 }883 if (ncur != nexp) {884 is_ok = false;885 }886 } else {887 for (size_t i = 0; i < GGML_MAX_DIMS; ++i) {888 if ((i < ne.size() && ne[i] != cur->ne[i]) || (i >= ne.size() && cur->ne[i] != 1)) {889 is_ok = false;890 break;891 }892 }893 }894 895 if (!is_ok) {896 throw std::runtime_error(897 format("%s: tensor '%s' has wrong shape; expected %s, got %s",898 __func__, name.c_str(),899 llama_format_tensor_shape(ne).c_str(),900 llama_format_tensor_shape(cur).c_str()));901 }902 903 return cur;904}905 906// checks if the weight tensor can be used with the specified buffer type and device907static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w, ggml_op op, ggml_backend_buffer_type_t buft, ggml_backend_dev_t dev) {908 GGML_ASSERT(w != nullptr);909 910 if (op == GGML_OP_NONE) {911 return true;912 }913 914 ggml_init_params params = {915 /*.mem_size =*/ ggml_tensor_overhead()*8,916 /*.mem_buffer =*/ NULL,917 /*.no_alloc =*/ true,918 };919 ggml_context_ptr ctx_ptr { ggml_init(params) };920 if (!ctx_ptr) {921 throw std::runtime_error(format("failed to create ggml context"));922 }923 ggml_context * ctx = ctx_ptr.get();924 925 ggml_tensor * op_tensor = nullptr;926 927 switch (op) {928 case GGML_OP_GET_ROWS:929 {930 ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);931 op_tensor = ggml_get_rows(ctx, w, b);932 } break;933 case GGML_OP_MUL_MAT:934 {935 ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], 512, w->ne[2], w->ne[3]);936 op_tensor = ggml_mul_mat(ctx, w, b);937 } break;938 case GGML_OP_MUL_MAT_ID:939 {940 // Used for either MoE expert routing or embedded adapter routing941 const int n_ids_used = hparams.router_layer >= 0 ? 1 : hparams.n_expert_used;942 GGML_ASSERT(n_ids_used > 0);943 ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_ids_used, 512);944 ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_ids_used, 512);945 op_tensor = ggml_mul_mat_id(ctx, w, b, ids);946 } break;947 case GGML_OP_ADD:948 {949 ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], w->ne[1], w->ne[2], w->ne[3]);950 op_tensor = ggml_add(ctx, a, w);951 } break;952 case GGML_OP_ADD_ID:953 {954 const int n_expert_used = hparams.n_expert_used;955 GGML_ASSERT(n_expert_used > 0);956 ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);957 ggml_tensor * c = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);958 op_tensor = ggml_add_id(ctx, a, w, c);959 } break;960 case GGML_OP_MUL:961 {962 ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], w->ne[1], w->ne[2], w->ne[3]);963 op_tensor = ggml_mul(ctx, a, w);964 } break;965 case GGML_OP_DIV:966 {967 ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, w->ne[0]);968 op_tensor = ggml_div(ctx, a, w);969 } break;970 case GGML_OP_ROPE:971 {972 const int n_embd_head = hparams.n_embd_head_v();973 const int n_head = hparams.n_head();974 ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_head, n_head, 512);975 ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);976 op_tensor = ggml_rope_ext(977 ctx, a, b, w,978 0, 0, 0, 0, 0,979 0, 0, 0, 0980 );981 982 } break;983 case GGML_OP_SSM_CONV:984 {985 const int64_t n_seq_tokens = 512;986 const int64_t n_seqs = 3;987 ggml_tensor * conv_x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0] - 1 + n_seq_tokens, w->ne[1], n_seqs);988 op_tensor = ggml_ssm_conv(ctx, conv_x, w);989 } break;990 case GGML_OP_SSM_SCAN:991 {992 // w is ssm_a, which is used to distinguish Mamba-1 and Mamba-2993 const int64_t d_state = w->ne[0] == 1 ? hparams.ssm_d_state : w->ne[0];994 const int64_t n_head = w->ne[1];995 const int64_t head_dim = hparams.ssm_d_inner / n_head;996 const int64_t n_group = hparams.ssm_n_group ? hparams.ssm_n_group : 1;997 const int64_t n_seq_tokens = 512;998 const int64_t n_seqs = 3;999 ggml_tensor * s = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, head_dim, n_head, n_seqs);1000 ggml_tensor * x = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, head_dim, n_head, n_seq_tokens, n_seqs);1001 ggml_tensor * dt = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_head, n_seq_tokens, n_seqs);1002 ggml_tensor * B = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, n_group, n_seq_tokens, n_seqs);1003 ggml_tensor * C = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, d_state, n_group, n_seq_tokens, n_seqs);1004 ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, n_seqs);1005 op_tensor = ggml_ssm_scan(ctx, s, x, dt, w, B, C, ids);1006 } break;1007 case GGML_OP_RWKV_WKV6:1008 {1009 // FIXME1010 const int64_t S = 123;1011 const int64_t H = 123;1012 const int64_t n_tokens = 123;1013 const int64_t n_seqs = 123;1014 ggml_tensor * k = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);1015 ggml_tensor * v = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);1016 ggml_tensor * r = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);1017 ggml_tensor * tf = w;1018 ggml_tensor * td = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);1019 ggml_tensor * state = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, S, n_seqs, S, H);1020 op_tensor = ggml_rwkv_wkv6(ctx, k, v, r, tf, td, state);1021 } break;1022 case GGML_OP_IM2COL:1023 {1024 const int n_embd_inp = hparams.n_embd_inp();1025 ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_embd_inp, w->ne[1], 1, 1);1026 op_tensor = ggml_im2col(ctx, w, b, 1, 0, 0, 0, 1, 0, false, GGML_TYPE_F16);1027 } break;1028 case GGML_OP_SCALE:1029 {1030 op_tensor = ggml_scale(ctx, w, 1.0f);1031 } break;1032 default:1033 GGML_ABORT("%s: missing test for op %s for tensor %s", __func__, ggml_op_name(op), w->name);1034 }1035 1036 // create a temporary dummy buffer for the weight so that supports_op can check the buffer type1037 GGML_ASSERT(w->buffer == nullptr);1038 w->buffer = ggml_backend_buft_alloc_buffer(buft, 0);1039 bool op_supported = ggml_backend_dev_supports_op(dev, op_tensor);1040 ggml_backend_buffer_free(w->buffer);1041 w->buffer = nullptr;1042 1043 return op_supported;1044}1045 1046// find the first buffer type in the list that can use the tensor1047static ggml_backend_buffer_type_t select_weight_buft(const llama_hparams & hparams, ggml_tensor * tensor, ggml_op op, const buft_list_t * buft_list) {1048 GGML_ASSERT(!buft_list->empty());1049 for (const auto & cur : *buft_list) {1050 ggml_backend_dev_t cur_dev = cur.first;1051 ggml_backend_buffer_type_t cur_buft = cur.second;1052 if (weight_buft_supported(hparams, tensor, op, cur_buft, cur_dev)) {1053 return cur_buft;1054 }1055 }1056 1057 return nullptr;1058}1059 1060struct ggml_tensor * llama_model_loader::create_tensor(1061 const llama_hparams & hparams, const buft_list_t * buft_list_cpu, const buft_list_t * buft_list_input, const buft_list_t * buft_list_output,1062 const buft_list_t * buft_list_layer, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {1063 auto ctx_for_buft = [&](ggml_backend_buffer_type_t buft) -> ggml_context * {1064 auto it = ctx_map.find(buft);1065 if (it == ctx_map.end()) {1066 // one ggml context per buffer type1067 int max_n_tensors = n_tensors;1068 max_n_tensors += 1; // duplicated output tensor1069 max_n_tensors += hparams.n_layer()*2; // duplicated rope freq tensors1070 if (files.empty()) {1071 max_n_tensors += hparams.n_layer()*256; // this should be well above what any model actually uses1072 }1073 const size_t ctx_size = ggml_tensor_overhead()*max_n_tensors;1074 1075 ggml_init_params params = {1076 /*.mem_size =*/ ctx_size,1077 /*.mem_buffer =*/ NULL,1078 /*.no_alloc =*/ true,1079 };1080 1081 ggml_context * ctx = ggml_init(params);1082 if (!ctx) {1083 throw std::runtime_error(format("failed to create ggml context"));1084 }1085 1086 ctx_map.emplace(buft, ctx);1087 1088 return ctx;1089 }1090 return it->second.get();1091 };1092 1093 auto buft_for_tensor = [&](ggml_tensor * t_meta) -> ggml_backend_buffer_type_t {1094 if (!t_meta) {1095 if (flags & TENSOR_NOT_REQUIRED) {1096 return nullptr;1097 }1098 throw std::runtime_error(format("missing tensor '%s'", tn.str().c_str()));1099 }1100 1101 // some models use the token embedding tensor as the output, but since these are used in different layers and with different ops1102 // the tensor is duplicated1103 // to handle this, we check if the tensor is duplicated, and if so, we assume that it is being loaded as the output tensor1104 llm_tensor tn_tensor = tn.tensor;1105 if (tn.tensor == LLM_TENSOR_TOKEN_EMBD && (flags & TENSOR_DUPLICATED)) {1106 tn_tensor = LLM_TENSOR_OUTPUT;1107 }1108 1109 llm_tensor_info info;1110 try {1111 info = llm_tensor_info_for(tn_tensor);1112 } catch (const std::out_of_range & e) {1113 throw std::runtime_error(format("missing tensor info mapping for %s", tn.str().c_str()));1114 }1115 1116 // skip unused tensors1117 if (info.op == GGML_OP_NONE || (flags & TENSOR_SKIP)) {1118 const size_t nbytes = ggml_nbytes(t_meta);1119 LLAMA_LOG_WARN("model has unused tensor %s (size = %zu bytes) -- ignoring\n", tn.str().c_str(), nbytes);1120 1121 size_data -= nbytes;1122 n_created++;1123 1124 return nullptr;1125 }1126 1127 // tensors with "bias" suffix are always used with GGML_OP_ADD or GGML_OP_ADD_ID;1128 // embedded-adapter ".lora_a"/".lora_b" tensors are always used with GGML_OP_MUL_MAT_ID1129 ggml_op op;1130 if (tn.suffix != nullptr && strcmp(tn.suffix, "bias") == 0) {1131 op = info.op == GGML_OP_MUL_MAT_ID ? GGML_OP_ADD_ID : GGML_OP_ADD;1132 } else if (hparams.router_layer >= 0 && tn.suffix != nullptr &&1133 (strcmp(tn.suffix, "lora_a") == 0 || strcmp(tn.suffix, "lora_b") == 0)) {1134 op = GGML_OP_MUL_MAT_ID;1135 } else {1136 op = info.op;1137 }1138 1139 // sanity checks1140 if (info.layer == LLM_TENSOR_LAYER_INPUT || info.layer == LLM_TENSOR_LAYER_OUTPUT) {1141 if (tn.bid != -1) {1142 GGML_ABORT("input/output layer tensor %s used with a layer number", tn.str().c_str());1143 }1144 } else {1145 if (tn.bid == -1) {1146 GGML_ABORT("repeating layer tensor %s used without a layer number", tn.str().c_str());1147 }1148 }1149 1150 // select the buffer type for this tensor1151 const buft_list_t * buft_list;1152 switch (info.layer) {1153 case LLM_TENSOR_LAYER_INPUT:1154 buft_list = buft_list_input;1155 break;1156 case LLM_TENSOR_LAYER_OUTPUT:1157 buft_list = buft_list_output;1158 break;1159 case LLM_TENSOR_LAYER_REPEATING:1160 GGML_ASSERT(buft_list_layer != nullptr);1161 buft_list = buft_list_layer;1162 break;1163 default:1164 GGML_ABORT("invalid layer %d for tensor %s", info.layer, tn.str().c_str());1165 }1166 1167 ggml_backend_buffer_type_t buft = nullptr;1168 1169 // check overrides1170 if (tensor_buft_overrides) {1171 std::string tensor_name = tn.str();1172 for (const auto * overrides = tensor_buft_overrides; overrides->pattern != nullptr; ++overrides) {1173 std::regex pattern(overrides->pattern);1174 if (std::regex_search(tensor_name, pattern)) {1175 if (overrides->buft == ggml_backend_cpu_buffer_type()) {1176 // when overriding to a CPU buffer, consider the extra buffer types1177 buft = select_weight_buft(hparams, t_meta, op, buft_list_cpu);1178 if (use_mmap) {1179 static std::once_flag once;1180 std::call_once(once, [] {1181 LLAMA_LOG_WARN("llama_model_loader: tensor overrides to CPU are used with mmap enabled - consider using --no-mmap for better performance\n");1182 });1183 }1184 } else {1185 buft = overrides->buft;1186 }1187 1188 LLAMA_LOG_DEBUG("tensor %s (%zu MiB %s) buffer type overridden to %s\n",1189 tensor_name.c_str(),1190 ggml_nbytes(t_meta) / 1024 / 1024, ggml_type_name(t_meta->type),1191 ggml_backend_buft_name(buft));1192 break;1193 }1194 }1195 }1196 1197 if (!buft) {1198 buft = select_weight_buft(hparams, t_meta, op, buft_list);1199 if (!buft) {1200 throw std::runtime_error(format("failed to find a compatible buffer type for tensor %s", tn.str().c_str()));