KBaba7/llama.cpp
0
1#include "llama-model.h"2 3#include "llama-impl.h"4#include "llama-mmap.h"5#include "llama-model-loader.h"6 7#include "ggml-cpp.h"8 9#include <algorithm>10#include <cassert>11#include <cstring>12#include <functional>13#include <map>14#include <sstream>15#include <stdexcept>16 17const char * llm_type_name(llm_type type) {18 switch (type) {19 case LLM_TYPE_14M: return "14M";20 case LLM_TYPE_17M: return "17M";21 case LLM_TYPE_22M: return "22M";22 case LLM_TYPE_33M: return "33M";23 case LLM_TYPE_60M: return "60M";24 case LLM_TYPE_70M: return "70M";25 case LLM_TYPE_80M: return "80M";26 case LLM_TYPE_109M: return "109M";27 case LLM_TYPE_137M: return "137M";28 case LLM_TYPE_160M: return "160M";29 case LLM_TYPE_220M: return "220M";30 case LLM_TYPE_250M: return "250M";31 case LLM_TYPE_270M: return "270M";32 case LLM_TYPE_335M: return "335M";33 case LLM_TYPE_410M: return "410M";34 case LLM_TYPE_450M: return "450M";35 case LLM_TYPE_770M: return "770M";36 case LLM_TYPE_780M: return "780M";37 case LLM_TYPE_0_5B: return "0.5B";38 case LLM_TYPE_1B: return "1B";39 case LLM_TYPE_1_3B: return "1.3B";40 case LLM_TYPE_1_4B: return "1.4B";41 case LLM_TYPE_1_5B: return "1.5B";42 case LLM_TYPE_1_6B: return "1.6B";43 case LLM_TYPE_2B: return "2B";44 case LLM_TYPE_2_8B: return "2.8B";45 case LLM_TYPE_3B: return "3B";46 case LLM_TYPE_4B: return "4B";47 case LLM_TYPE_6B: return "6B";48 case LLM_TYPE_6_9B: return "6.9B";49 case LLM_TYPE_7B: return "7B";50 case LLM_TYPE_8B: return "8B";51 case LLM_TYPE_9B: return "9B";52 case LLM_TYPE_11B: return "11B";53 case LLM_TYPE_12B: return "12B";54 case LLM_TYPE_13B: return "13B";55 case LLM_TYPE_14B: return "14B";56 case LLM_TYPE_15B: return "15B";57 case LLM_TYPE_16B: return "16B";58 case LLM_TYPE_20B: return "20B";59 case LLM_TYPE_30B: return "30B";60 case LLM_TYPE_32B: return "32B";61 case LLM_TYPE_34B: return "34B";62 case LLM_TYPE_35B: return "35B";63 case LLM_TYPE_40B: return "40B";64 case LLM_TYPE_65B: return "65B";65 case LLM_TYPE_70B: return "70B";66 case LLM_TYPE_236B: return "236B";67 case LLM_TYPE_314B: return "314B";68 case LLM_TYPE_671B: return "671B";69 case LLM_TYPE_SMALL: return "0.1B";70 case LLM_TYPE_MEDIUM: return "0.4B";71 case LLM_TYPE_LARGE: return "0.8B";72 case LLM_TYPE_XL: return "1.5B";73 case LLM_TYPE_A1_7B: return "A1.7B";74 case LLM_TYPE_A2_7B: return "A2.7B";75 case LLM_TYPE_8x7B: return "8x7B";76 case LLM_TYPE_8x22B: return "8x22B";77 case LLM_TYPE_16x12B: return "16x12B";78 case LLM_TYPE_16x3_8B: return "16x3.8B";79 case LLM_TYPE_10B_128x3_66B: return "10B+128x3.66B";80 case LLM_TYPE_57B_A14B: return "57B.A14B";81 case LLM_TYPE_27B: return "27B";82 default: return "?B";83 }84}85 86static const char * llama_expert_gating_func_name(llama_expert_gating_func_type type) {87 switch (type) {88 case LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX: return "softmax";89 case LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID: return "sigmoid";90 default: return "unknown";91 }92}93 94static const std::map<llama_rope_scaling_type, const char *> LLAMA_ROPE_SCALING_TYPES = {95 { LLAMA_ROPE_SCALING_TYPE_NONE, "none" },96 { LLAMA_ROPE_SCALING_TYPE_LINEAR, "linear" },97 { LLAMA_ROPE_SCALING_TYPE_YARN, "yarn" },98 { LLAMA_ROPE_SCALING_TYPE_LONGROPE, "longrope" },99};100 101static llama_rope_scaling_type llama_rope_scaling_type_from_string(const std::string & name) {102 for (const auto & kv : LLAMA_ROPE_SCALING_TYPES) {103 if (kv.second == name) {104 return (llama_rope_scaling_type) kv.first;105 }106 }107 108 return LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;109}110 111// checks if the weight tensor can be used with the specified buffer type and device112static bool weight_buft_supported(const llama_hparams & hparams, ggml_tensor * w, ggml_op op, ggml_backend_buffer_type_t buft, ggml_backend_dev_t dev) {113 GGML_ASSERT(w != nullptr);114 115 if (op == GGML_OP_NONE) {116 return true;117 }118 119 ggml_init_params params = {120 /*.mem_size =*/ ggml_tensor_overhead()*8,121 /*.mem_buffer =*/ NULL,122 /*.no_alloc =*/ true,123 };124 ggml_context_ptr ctx_ptr { ggml_init(params) };125 if (!ctx_ptr) {126 throw std::runtime_error(format("failed to create ggml context"));127 }128 ggml_context * ctx = ctx_ptr.get();129 130 ggml_tensor * op_tensor = nullptr;131 132 switch (op) {133 case GGML_OP_GET_ROWS:134 {135 ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);136 op_tensor = ggml_get_rows(ctx, w, b);137 } break;138 case GGML_OP_MUL_MAT:139 {140 ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], 512, w->ne[2], w->ne[3]);141 op_tensor = ggml_mul_mat(ctx, w, b);142 } break;143 case GGML_OP_MUL_MAT_ID:144 {145 int n_expert_used = hparams.n_expert_used;146 ggml_tensor * b = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, w->ne[0], n_expert_used, 512);147 ggml_tensor * ids = ggml_new_tensor_2d(ctx, GGML_TYPE_I32, n_expert_used, 512);148 op_tensor = ggml_mul_mat_id(ctx, w, b, ids);149 } break;150 case GGML_OP_ADD:151 {152 ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], w->ne[1], w->ne[2], w->ne[3]);153 op_tensor = ggml_add(ctx, a, w);154 } break;155 case GGML_OP_MUL:156 {157 ggml_tensor * a = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, w->ne[0], w->ne[1], w->ne[2], w->ne[3]);158 op_tensor = ggml_mul(ctx, a, w);159 } break;160 case GGML_OP_DIV:161 {162 ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, w->ne[0]);163 op_tensor = ggml_div(ctx, a, w);164 } break;165 case GGML_OP_ROPE:166 {167 int n_embd_head = hparams.n_embd_head_v;168 int n_head = hparams.n_head();169 ggml_tensor * a = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, n_embd_head, n_head, 512);170 ggml_tensor * b = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 512);171 op_tensor = ggml_rope_ext(172 ctx, a, b, w,173 0, 0, 0, 0, 0,174 0, 0, 0, 0175 );176 177 } break;178 case GGML_OP_SSM_CONV:179 {180 // FIXME181 ggml_tensor * conv_x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 12345, w->ne[1], 6789);182 op_tensor = ggml_ssm_conv(ctx, conv_x, w);183 } break;184 case GGML_OP_SSM_SCAN:185 {186 // FIXME187 const int64_t d_state = w->ne[0];188 const int64_t d_inner = w->ne[1];189 const int64_t n_seq_tokens = 512;190 const int64_t n_seqs = 1;191 ggml_tensor * s = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, d_state, d_inner, n_seqs);192 ggml_tensor * x = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, d_inner, n_seq_tokens, n_seqs);193 ggml_tensor * dt = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, d_inner, n_seq_tokens, n_seqs);194 ggml_tensor * B = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, d_state, n_seq_tokens, n_seqs);195 ggml_tensor * C = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, d_state, n_seq_tokens, n_seqs);196 op_tensor = ggml_ssm_scan(ctx, s, x, dt, w, B, C);197 } break;198 case GGML_OP_RWKV_WKV6:199 {200 // FIXME201 const int64_t S = 123;202 const int64_t H = 123;203 const int64_t n_tokens = 123;204 const int64_t n_seqs = 123;205 ggml_tensor * k = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);206 ggml_tensor * v = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);207 ggml_tensor * r = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);208 ggml_tensor * tf = w;209 ggml_tensor * td = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, S, H, n_tokens);210 ggml_tensor * state = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, S, n_seqs, S, H);211 op_tensor = ggml_rwkv_wkv6(ctx, k, v, r, tf, td, state);212 } break;213 case GGML_OP_IM2COL:214 {215 const int n_embd = hparams.n_embd;216 ggml_tensor * b = ggml_new_tensor_4d(ctx, GGML_TYPE_F32, n_embd, w->ne[1], 1, 1);217 op_tensor = ggml_im2col(ctx, w, b, 1, 0, 0, 0, 1, 0, false, GGML_TYPE_F16);218 } break;219 default:220 GGML_ABORT("%s: missing test for op %s for tensor %s", __func__, ggml_op_name(op), w->name);221 }222 223 // create a temporary dummy buffer for the weight so that supports_op can check the buffer type224 GGML_ASSERT(w->buffer == nullptr);225 w->buffer = ggml_backend_buft_alloc_buffer(buft, 0);226 bool op_supported = ggml_backend_dev_supports_op(dev, op_tensor);227 ggml_backend_buffer_free(w->buffer);228 w->buffer = nullptr;229 230 return op_supported;231}232 233// lists of buffer types used for each layer234using buft_list_t = std::vector<std::pair<ggml_backend_dev_t, ggml_backend_buffer_type_t>>;235 236// find the first buffer type in the list that can use the tensor237static ggml_backend_buffer_type_t select_weight_buft(const llama_hparams & hparams, ggml_tensor * tensor, ggml_op op, const buft_list_t & buft_list) {238 GGML_ASSERT(!buft_list.empty());239 for (const auto & cur : buft_list) {240 ggml_backend_dev_t cur_dev = cur.first;241 ggml_backend_buffer_type_t cur_buft = cur.second;242 if (weight_buft_supported(hparams, tensor, op, cur_buft, cur_dev)) {243 return cur_buft;244 }245 }246 return nullptr;247}248 249// CPU: ACCEL -> CPU extra -> GPU host -> CPU250static buft_list_t make_cpu_buft_list(const std::vector<ggml_backend_dev_t> & devices) {251 buft_list_t buft_list;252 253 // add ACCEL buffer types254 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {255 ggml_backend_dev_t dev = ggml_backend_dev_get(i);256 if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {257 auto * buft = ggml_backend_dev_buffer_type(dev);258 // skip259 if (buft != ggml_backend_cpu_buffer_type()) {260 buft_list.emplace_back(dev, buft);261 }262 }263 }264 265 // add extra buffer types266 auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);267 auto * cpu_reg = ggml_backend_dev_backend_reg(cpu_dev);268 auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)269 ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_dev_get_extra_bufts");270 if (ggml_backend_dev_get_extra_bufts_fn) {271 ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(cpu_dev);272 while (extra_bufts && *extra_bufts) {273 buft_list.emplace_back(cpu_dev, *extra_bufts);274 ++extra_bufts;275 }276 }277 278 // add a host buffer type279 // storing the tensors in a host buffer is useful when the processing of large batches280 // is offloaded to a GPU device, since it reduces the time spent on data transfers281 // generally, this will be done using the first device in the list282 // a better approach would be to handle this on a weight-by-weight basis using the offload_op283 // function of the device to determine if it would benefit from being stored in a host buffer284 for (auto * dev : devices) {285 ggml_backend_buffer_type_t buft = ggml_backend_dev_host_buffer_type(dev);286 if (buft) {287 buft_list.emplace_back(dev, buft);288 break;289 }290 }291 292 // add the CPU buffer type293 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {294 ggml_backend_dev_t dev = ggml_backend_dev_get(i);295 if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {296 buft_list.emplace_back(dev, ggml_backend_dev_buffer_type(dev));297 }298 }299 300 return buft_list;301}302 303// GPU: split if LLAMA_SPLIT_MODE_ROW -> GPU304static buft_list_t make_gpu_buft_list(ggml_backend_dev_t dev, enum llama_split_mode split_mode, const float * tensor_split) {305 buft_list_t buft_list;306 307 // add the device split buffer type if requested and available308 if (split_mode == LLAMA_SPLIT_MODE_ROW) {309 ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);310 auto ggml_backend_split_buffer_type_fn = (ggml_backend_split_buffer_type_t)311 ggml_backend_reg_get_proc_address(reg, "ggml_backend_split_buffer_type");312 if (ggml_backend_split_buffer_type_fn) {313 size_t dev_index = [&]() {314 auto * reg = ggml_backend_dev_backend_reg(dev);315 for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); ++i) {316 if (ggml_backend_reg_dev_get(reg, i) == dev) {317 return i;318 }319 }320 throw std::runtime_error(format("device %s not found in its backend reg", ggml_backend_dev_name(dev)));321 }();322 auto * buft = ggml_backend_split_buffer_type_fn(dev_index, tensor_split);323 if (buft != nullptr) {324 buft_list.emplace_back(dev, buft);325 }326 }327 }328 329 // add the device default buffer type330 buft_list.emplace_back(dev, ggml_backend_dev_buffer_type(dev));331 332 return buft_list;333}334 335struct llama_model::impl {336 impl() {}337 ~impl() {}338 339 uint64_t n_elements = 0;340 341 size_t n_bytes = 0;342 343 std::string desc_str;344 345 // model memory mapped files346 llama_mmaps mappings;347 348 // objects representing data potentially being locked in memory349 llama_mlocks mlock_bufs;350 llama_mlocks mlock_mmaps;351 352 // contexts where the model tensors metadata is stored353 std::vector<ggml_context_ptr> ctxs;354 355 // the model memory buffers for the tensor data356 std::vector<ggml_backend_buffer_ptr> bufs;357 358 buft_list_t cpu_buft_list;359 std::map<ggml_backend_dev_t, buft_list_t> gpu_buft_list;360 361 struct layer_dev {362 ggml_backend_dev_t dev;363 buft_list_t * buft_list;364 };365 366 layer_dev dev_input = {};367 layer_dev dev_output = {};368 std::vector<layer_dev> dev_layer;369};370 371llama_model::llama_model(const struct llama_model_params & params) : params(params), pimpl(std::make_unique<impl>()) {372}373 374llama_model::~llama_model() {}375 376void llama_model::load_stats(llama_model_loader & ml) {377 pimpl->n_elements = ml.n_elements;378 pimpl->n_bytes = ml.n_bytes;379}380 381void llama_model::load_arch(llama_model_loader & ml) {382 arch = ml.get_arch();383 if (arch == LLM_ARCH_UNKNOWN) {384 throw std::runtime_error("unknown model architecture: '" + ml.get_arch_name() + "'");385 }386}387 388void llama_model::load_hparams(llama_model_loader & ml) {389 const gguf_context * ctx = ml.meta.get();390 391 // get metadata as string392 for (int i = 0; i < gguf_get_n_kv(ctx); i++) {393 enum gguf_type type = gguf_get_kv_type(ctx, i);394 if (type == GGUF_TYPE_ARRAY) {395 continue;396 }397 const char * name = gguf_get_key(ctx, i);398 const std::string value = gguf_kv_to_str(ctx, i);399 gguf_kv.emplace(name, value);400 }401 402 // get general kv403 ml.get_key(LLM_KV_GENERAL_NAME, name, false);404 405 // everything past this point is not vocab-related406 if (hparams.vocab_only) {407 return;408 }409 410 ml.get_key(LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train);411 ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd);412 ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer);413 ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);414 ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);415 416 if (arch == LLM_ARCH_WAVTOKENIZER_DEC) {417 ml.get_key(LLM_KV_FEATURES_LENGTH, hparams.n_embd_features);418 419 ml.get_key(LLM_KV_POSNET_EMBEDDING_LENGTH, hparams.posnet.n_embd);420 ml.get_key(LLM_KV_POSNET_BLOCK_COUNT, hparams.posnet.n_layer);421 422 ml.get_key(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, hparams.convnext.n_embd);423 ml.get_key(LLM_KV_CONVNEXT_BLOCK_COUNT, hparams.convnext.n_layer);424 }425 426 GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS);427 GGML_ASSERT(hparams.n_expert_used <= hparams.n_expert);428 if (hparams.n_expert > 0) {429 GGML_ASSERT(hparams.n_expert_used > 0);430 } else {431 GGML_ASSERT(hparams.n_expert_used == 0);432 }433 434 // zero-out the array hparams435 std::fill(hparams.n_head_arr.begin(), hparams.n_head_arr.end(), 0);436 std::fill(hparams.n_head_kv_arr.begin(), hparams.n_head_kv_arr.end(), 0);437 std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0);438 439 ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer, false);440 ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer, false);441 442 // n_head_kv is optional, default to n_head443 hparams.n_head_kv_arr = hparams.n_head_arr;444 445 ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, hparams.n_layer, false);446 447 bool rope_finetuned = false;448 ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);449 hparams.rope_finetuned = rope_finetuned;450 451 hparams.n_ctx_orig_yarn = hparams.n_ctx_train;452 ml.get_key(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, hparams.n_ctx_orig_yarn, false);453 454 // rope_freq_base (optional)455 hparams.rope_freq_base_train = 10000.0f;456 ml.get_key(LLM_KV_ROPE_FREQ_BASE, hparams.rope_freq_base_train, false);457 458 std::string rope_scaling("linear");459 ml.get_key(LLM_KV_ROPE_SCALING_TYPE, rope_scaling, false);460 hparams.rope_scaling_type_train = llama_rope_scaling_type_from_string(rope_scaling);461 GGML_ASSERT(hparams.rope_scaling_type_train != LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED);462 463 // rope_freq_scale (inverse of the kv) is optional464 float ropescale = 0.0f;465 if (!ml.get_key(LLM_KV_ROPE_SCALING_FACTOR, ropescale, false)) {466 // try the old key name467 ml.get_key(LLM_KV_ROPE_SCALE_LINEAR, ropescale, false);468 }469 hparams.rope_freq_scale_train = ropescale == 0.0f ? 1.0f : 1.0f/ropescale;470 471 ml.get_key(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor, false);472 473 // non-transformer models do not have attention heads474 if (hparams.n_head() > 0) {475 // gpt-neox n_rot = rotary_pct * (n_embd / n_head)476 // gpt-j n_rot = rotary_dim477 478 hparams.n_embd_head_k = hparams.n_embd / hparams.n_head();479 ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k, false);480 481 hparams.n_embd_head_v = hparams.n_embd / hparams.n_head();482 ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v, false);483 484 // sanity check for n_rot (optional)485 hparams.n_rot = hparams.n_embd_head_k;486 487 ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT, hparams.n_rot, false);488 489 if (arch == LLM_ARCH_LLAMA || arch == LLM_ARCH_DECI || arch == LLM_ARCH_FALCON) {490 if (hparams.n_rot != hparams.n_embd_head_k) {491 throw std::runtime_error(format("invalid n_rot: %u, expected %u", hparams.n_rot, hparams.n_embd_head_k));492 }493 }494 } else {495 hparams.n_rot = 0;496 hparams.n_embd_head_k = 0;497 hparams.n_embd_head_v = 0;498 }499 500 // for differentiating model types501 uint32_t n_vocab = 0;502 ml.get_key(LLM_KV_VOCAB_SIZE, n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, n_vocab, false);503 504 // arch-specific KVs505 switch (arch) {506 case LLM_ARCH_LLAMA:507 {508 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);509 510 if (hparams.n_expert == 8) {511 switch (hparams.n_layer) {512 case 32: type = LLM_TYPE_8x7B; break;513 case 56: type = LLM_TYPE_8x22B; break;514 default: type = LLM_TYPE_UNKNOWN;515 }516 } else {517 switch (hparams.n_layer) {518 case 16: type = LLM_TYPE_1B; break; // Llama 3.2 1B519 case 22: type = LLM_TYPE_1B; break;520 case 26: type = LLM_TYPE_3B; break;521 case 28: type = LLM_TYPE_3B; break; // Llama 3.2 3B522 // granite uses a vocab with len 49152523 case 32: type = n_vocab == 49152 ? LLM_TYPE_3B : (n_vocab < 40000 ? LLM_TYPE_7B : LLM_TYPE_8B); break;524 case 36: type = LLM_TYPE_8B; break; // granite525 case 40: type = LLM_TYPE_13B; break;526 case 48: type = LLM_TYPE_34B; break;527 case 60: type = LLM_TYPE_30B; break;528 case 80: type = hparams.n_head() == hparams.n_head_kv() ? LLM_TYPE_65B : LLM_TYPE_70B; break;529 default: type = LLM_TYPE_UNKNOWN;530 }531 }532 } break;533 case LLM_ARCH_DECI:534 {535 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);536 switch (hparams.n_layer) {537 case 32: type = LLM_TYPE_7B; break;538 case 80: type = LLM_TYPE_70B; break;539 default: type = LLM_TYPE_UNKNOWN;540 }541 } break;542 case LLM_ARCH_MINICPM:543 {544 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);545 ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale);546 ml.get_key(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale);547 ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);548 549 switch (hparams.n_layer) {550 case 52: type = LLM_TYPE_1B; break;551 case 40: type = LLM_TYPE_2B; break;552 default: type = LLM_TYPE_UNKNOWN;553 }554 } break;555 case LLM_ARCH_MINICPM3:556 {557 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);558 ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);559 ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);560 561 switch (hparams.n_layer) {562 case 62: type = LLM_TYPE_4B; break;563 default: type = LLM_TYPE_UNKNOWN;564 }565 } break;566 case LLM_ARCH_GROK:567 {568 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);569 570 switch (hparams.n_layer) {571 case 64: type = LLM_TYPE_314B; break;572 default: type = LLM_TYPE_UNKNOWN;573 }574 } break;575 case LLM_ARCH_FALCON:576 {577 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);578 579 switch (hparams.n_layer) {580 case 32: type = LLM_TYPE_7B; break;581 case 60: type = LLM_TYPE_40B; break;582 default: type = LLM_TYPE_UNKNOWN;583 }584 } break;585 case LLM_ARCH_BAICHUAN:586 {587 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);588 switch (hparams.n_layer) {589 case 32: type = LLM_TYPE_7B; break;590 case 40: type = LLM_TYPE_13B; break;591 default: type = LLM_TYPE_UNKNOWN;592 }593 594 if (type == LLM_TYPE_13B) {595 // TODO: become GGUF KV parameter596 hparams.f_max_alibi_bias = 8.0f;597 }598 } break;599 case LLM_ARCH_STARCODER:600 {601 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);602 switch (hparams.n_layer) {603 case 24: type = LLM_TYPE_1B; break;604 case 36: type = LLM_TYPE_3B; break;605 case 42: type = LLM_TYPE_7B; break;606 case 40: type = LLM_TYPE_15B; break;607 default: type = LLM_TYPE_UNKNOWN;608 }609 } break;610 case LLM_ARCH_REFACT:611 {612 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);613 switch (hparams.n_layer) {614 case 32: type = LLM_TYPE_1B; break;615 default: type = LLM_TYPE_UNKNOWN;616 }617 618 // TODO: become GGUF KV parameter619 hparams.f_max_alibi_bias = 8.0f;620 } break;621 case LLM_ARCH_BERT:622 {623 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);624 ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);625 ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);626 627 switch (hparams.n_layer) {628 case 3:629 type = LLM_TYPE_17M; break; // bge-micro630 case 6:631 type = LLM_TYPE_22M; break; // MiniLM-L6632 case 12:633 switch (hparams.n_embd) {634 case 384: type = LLM_TYPE_33M; break; // MiniLM-L12, bge-small635 case 768: type = LLM_TYPE_109M; break; // bge-base636 default: type = LLM_TYPE_UNKNOWN;637 } break;638 case 24:639 type = LLM_TYPE_335M; break; // bge-large640 default: type = LLM_TYPE_UNKNOWN;641 }642 } break;643 case LLM_ARCH_JINA_BERT_V2:644 {645 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);646 ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);647 ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);648 hparams.f_max_alibi_bias = 8.0f;649 650 switch (hparams.n_layer) {651 case 4: type = LLM_TYPE_33M; break; // jina-embeddings-small652 case 12: type = LLM_TYPE_137M; break; // jina-embeddings-base653 default: type = LLM_TYPE_UNKNOWN;654 }655 } break;656 case LLM_ARCH_NOMIC_BERT:657 {658 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);659 ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);660 ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type);661 662 if (hparams.n_layer == 12 && hparams.n_embd == 768) {663 type = LLM_TYPE_137M;664 }665 } break;666 case LLM_ARCH_BLOOM:667 {668 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);669 670 switch (hparams.n_layer) {671 case 24: type = LLM_TYPE_1B; break;672 case 30:673 switch (hparams.n_embd) {674 case 2560: type = LLM_TYPE_3B; break;675 case 4096: type = LLM_TYPE_7B; break;676 default: type = LLM_TYPE_UNKNOWN;677 } break;678 default: type = LLM_TYPE_UNKNOWN;679 }680 681 // TODO: become GGUF KV parameter682 hparams.f_max_alibi_bias = 8.0f;683 } break;684 case LLM_ARCH_MPT:685 {686 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);687 ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv, false);688 ml.get_key(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, hparams.f_max_alibi_bias);689 690 switch (hparams.n_layer) {691 case 32: type = LLM_TYPE_7B; break;692 case 48: type = LLM_TYPE_30B; break;693 default: type = LLM_TYPE_UNKNOWN;694 }695 } break;696 case LLM_ARCH_STABLELM:697 {698 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);699 700 switch (hparams.n_layer) {701 case 24: type = LLM_TYPE_1B; break;702 case 32: type = LLM_TYPE_3B; break;703 case 40: type = LLM_TYPE_12B; break;704 default: type = LLM_TYPE_UNKNOWN;705 }706 } break;707 case LLM_ARCH_QWEN:708 {709 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);710 711 switch (hparams.n_layer) {712 case 32: type = LLM_TYPE_7B; break;713 case 40: type = LLM_TYPE_13B; break;714 default: type = LLM_TYPE_UNKNOWN;715 }716 } break;717 case LLM_ARCH_QWEN2VL:718 {719 ml.get_key_or_arr(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections, 4, true);720 }721 // fall through722 case LLM_ARCH_QWEN2:723 {724 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);725 switch (hparams.n_layer) {726 case 24: type = hparams.n_embd == 1024 ? LLM_TYPE_0_5B : LLM_TYPE_1B; break;727 case 28: type = hparams.n_embd == 1536 ? LLM_TYPE_1_5B : LLM_TYPE_7B; break;728 case 32: type = LLM_TYPE_7B; break;729 case 36: type = LLM_TYPE_3B; break;730 case 40: type = hparams.n_head() == 20 ? LLM_TYPE_4B : LLM_TYPE_13B; break;731 case 48: type = LLM_TYPE_14B; break;732 case 64: type = LLM_TYPE_32B; break;733 case 80: type = LLM_TYPE_70B; break;734 default: type = LLM_TYPE_UNKNOWN;735 }736 } break;737 case LLM_ARCH_QWEN2MOE:738 {739 ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);740 ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false);741 742 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);743 switch (hparams.n_layer) {744 case 24: type = LLM_TYPE_A2_7B; break;745 case 28: type = LLM_TYPE_57B_A14B; break;746 default: type = LLM_TYPE_UNKNOWN;747 }748 } break;749 case LLM_ARCH_PHI2:750 {751 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);752 753 switch (hparams.n_layer) {754 case 24: type = LLM_TYPE_1B; break;755 case 32: type = LLM_TYPE_3B; break;756 default: type = LLM_TYPE_UNKNOWN;757 }758 } break;759 case LLM_ARCH_PHI3:760 {761 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);762 763 switch (hparams.n_layer) {764 case 24: type = LLM_TYPE_1B; break;765 case 32: type = LLM_TYPE_3B; break;766 case 40: type = LLM_TYPE_14B; break;767 default: type = LLM_TYPE_UNKNOWN;768 }769 770 // for backward compatibility ; see: https://github.com/ggerganov/llama.cpp/pull/8931771 if ((hparams.n_layer == 32 || hparams.n_layer == 40) && hparams.n_ctx_train == 4096) {772 // default value for Phi-3-mini-4k-instruct and Phi-3-medium-4k-instruct773 hparams.n_swa = 2047;774 } else if (hparams.n_layer == 32 && hparams.n_head_kv(0) == 32 && hparams.n_ctx_train == 131072) {775 // default value for Phi-3-mini-128k-instruct776 hparams.n_swa = 262144;777 } else if (hparams.n_layer == 40 && hparams.n_ctx_train == 131072) {778 // default value for Phi-3-medium-128k-instruct779 hparams.n_swa = 131072;780 }781 bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);782 if (!found_swa && hparams.n_swa == 0) {783 throw std::runtime_error("invalid value for sliding_window");784 }785 } break;786 case LLM_ARCH_PHIMOE:787 {788 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);789 790 switch (hparams.n_layer) {791 case 32: type = LLM_TYPE_16x3_8B; break;792 default: type = LLM_TYPE_UNKNOWN;793 }794 } break;795 case LLM_ARCH_PLAMO:796 {797 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);798 799 switch (hparams.n_layer) {800 case 40: type = LLM_TYPE_13B; break;801 default: type = LLM_TYPE_UNKNOWN;802 }803 } break;804 case LLM_ARCH_GPT2:805 {806 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);807 switch (hparams.n_layer) {808 case 12: type = LLM_TYPE_SMALL; break;809 case 24: type = LLM_TYPE_MEDIUM; break;810 case 36: type = LLM_TYPE_LARGE; break;811 case 48: type = LLM_TYPE_XL; break;812 default: type = LLM_TYPE_UNKNOWN;813 }814 } break;815 case LLM_ARCH_CODESHELL:816 {817 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);818 switch (hparams.n_layer) {819 case 42: type = LLM_TYPE_7B; break;820 default: type = LLM_TYPE_UNKNOWN;821 }822 } break;823 case LLM_ARCH_ORION:824 {825 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);826 827 switch (hparams.n_layer) {828 case 40: type = LLM_TYPE_14B; break;829 default: type = LLM_TYPE_UNKNOWN;830 }831 } break;832 case LLM_ARCH_INTERNLM2:833 {834 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);835 switch (hparams.n_layer) {836 case 32: type = LLM_TYPE_7B; break;837 case 48: type = LLM_TYPE_20B; break;838 default: type = LLM_TYPE_UNKNOWN;839 }840 } break;841 case LLM_ARCH_GEMMA:842 {843 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);844 845 switch (hparams.n_layer) {846 case 18: type = LLM_TYPE_2B; break;847 case 28: type = LLM_TYPE_7B; break;848 default: type = LLM_TYPE_UNKNOWN;849 }850 } break;851 case LLM_ARCH_GEMMA2:852 {853 hparams.n_swa = 4096; // default value of gemma 2854 ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);855 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);856 ml.get_key(LLM_KV_ATTN_LOGIT_SOFTCAPPING, hparams.f_attn_logit_softcapping, false);857 ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false);858 hparams.attn_soft_cap = true;859 860 switch (hparams.n_layer) {861 case 26: type = LLM_TYPE_2B; break;862 case 42: type = LLM_TYPE_9B; break;863 case 46: type = LLM_TYPE_27B; break;864 default: type = LLM_TYPE_UNKNOWN;865 }866 } break;867 case LLM_ARCH_STARCODER2:868 {869 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);870 switch (hparams.n_layer) {871 case 30: type = LLM_TYPE_3B; break;872 case 32: type = LLM_TYPE_7B; break;873 case 40: type = LLM_TYPE_15B; break;874 case 52: type = LLM_TYPE_20B; break; // granite875 case 88: type = LLM_TYPE_34B; break; // granite876 default: type = LLM_TYPE_UNKNOWN;877 }878 } break;879 case LLM_ARCH_MAMBA:880 {881 ml.get_key(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);882 ml.get_key(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);883 ml.get_key(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);884 ml.get_key(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);885 ml.get_key(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms, false);886 887 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);888 889 switch (hparams.n_layer) {890 case 24:891 switch (hparams.n_embd) {892 case 768: type = LLM_TYPE_SMALL; break;893 default: type = LLM_TYPE_UNKNOWN;894 } break;895 case 48:896 switch (hparams.n_embd) {897 case 1024: type = LLM_TYPE_MEDIUM; break;898 case 1536: type = LLM_TYPE_LARGE; break;899 case 2048: type = LLM_TYPE_XL; break;900 default: type = LLM_TYPE_UNKNOWN;901 } break;902 case 64:903 switch (hparams.n_embd) {904 case 2560: type = LLM_TYPE_3B; break;905 default: type = LLM_TYPE_UNKNOWN;906 } break;907 default: type = LLM_TYPE_UNKNOWN;908 }909 } break;910 case LLM_ARCH_XVERSE:911 {912 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);913 switch (hparams.n_layer) {914 case 32: type = LLM_TYPE_7B; break;915 case 40: type = LLM_TYPE_13B; break;916 case 80: type = LLM_TYPE_65B; break;917 default: type = LLM_TYPE_UNKNOWN;918 }919 } break;920 case LLM_ARCH_COMMAND_R:921 {922 ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);923 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);924 switch (hparams.n_layer) {925 case 40: type = LLM_TYPE_35B; break;926 default: type = LLM_TYPE_UNKNOWN;927 }928 } break;929 case LLM_ARCH_COHERE2:930 {931 ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);932 ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);933 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);934 switch (hparams.n_layer) {935 case 32: type = LLM_TYPE_8B; break;936 default: type = LLM_TYPE_UNKNOWN;937 }938 } break;939 case LLM_ARCH_DBRX:940 {941 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);942 ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv);943 944 switch (hparams.n_layer) {945 case 40: type = LLM_TYPE_16x12B; break;946 default: type = LLM_TYPE_UNKNOWN;947 }948 } break;949 case LLM_ARCH_OLMO:950 {951 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);952 ml.get_key(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv, false);953 954 switch (hparams.n_layer) {955 case 22: type = LLM_TYPE_1B; break;956 case 32: type = LLM_TYPE_7B; break;957 case 80: type = LLM_TYPE_70B; break;958 default: type = LLM_TYPE_UNKNOWN;959 }960 } break;961 case LLM_ARCH_OLMO2:962 {963 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);964 965 switch (hparams.n_layer) {966 case 16: type = LLM_TYPE_1B; break;967 case 32: type = LLM_TYPE_7B; break;968 case 40: type = LLM_TYPE_13B; break;969 default: type = LLM_TYPE_UNKNOWN;970 }971 } break;972 case LLM_ARCH_OLMOE:973 {974 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);975 switch (hparams.n_layer) {976 case 16: type = LLM_TYPE_A1_7B; break;977 default: type = LLM_TYPE_UNKNOWN;978 }979 } break;980 case LLM_ARCH_OPENELM:981 {982 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);983 984 switch (hparams.n_layer) {985 case 16: type = LLM_TYPE_270M; break;986 case 20: type = LLM_TYPE_450M; break;987 case 28: type = LLM_TYPE_1B; break;988 case 36: type = LLM_TYPE_3B; break;989 default: type = LLM_TYPE_UNKNOWN;990 }991 } break;992 case LLM_ARCH_GPTNEOX:993 {994 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);995 ml.get_key(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res);996 switch (hparams.n_layer) {997 case 6:998 switch (hparams.n_ff()) {999 case 512: type = LLM_TYPE_14M; break;1000 case 2048: type = LLM_TYPE_70M; break;1001 default: type = LLM_TYPE_UNKNOWN;1002 } break;1003 case 12:1004 switch (hparams.n_ff()) {1005 case 3072: type = LLM_TYPE_160M; break;1006 default: type = LLM_TYPE_UNKNOWN;1007 } break;1008 case 16:1009 switch (hparams.n_ff()) {1010 case 8192: type = LLM_TYPE_1B; break;1011 default: type = LLM_TYPE_UNKNOWN;1012 } break;1013 case 24:1014 switch (hparams.n_ff()) {1015 case 4096: type = LLM_TYPE_410M; break;1016 case 8192: type = LLM_TYPE_1_4B; break;1017 default: type = LLM_TYPE_UNKNOWN;1018 } break;1019 case 32:1020 switch (hparams.n_ff()) {1021 case 10240: type = LLM_TYPE_2_8B; break;1022 case 16384: type = LLM_TYPE_6_9B; break;1023 default: type = LLM_TYPE_UNKNOWN;1024 } break;1025 case 36:1026 switch (hparams.n_ff()) {1027 case 20480: type = LLM_TYPE_12B; break;1028 default: type = LLM_TYPE_UNKNOWN;1029 } break;1030 case 44:1031 switch (hparams.n_ff()) {1032 case 24576: type = LLM_TYPE_20B; break;1033 default: type = LLM_TYPE_UNKNOWN;1034 } break;1035 default: type = LLM_TYPE_UNKNOWN;1036 }1037 } break;1038 case LLM_ARCH_ARCTIC:1039 {1040 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1041 1042 if (hparams.n_expert == 128) {1043 switch (hparams.n_layer) {1044 case 35: type = LLM_TYPE_10B_128x3_66B; break;1045 default: type = LLM_TYPE_UNKNOWN;1046 }1047 } else {1048 type = LLM_TYPE_UNKNOWN;1049 }1050 } break;1051 case LLM_ARCH_DEEPSEEK:1052 {1053 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1054 ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);1055 ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);1056 ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);1057 ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);1058 1059 switch (hparams.n_layer) {1060 case 28: type = LLM_TYPE_20B; break;1061 default: type = LLM_TYPE_UNKNOWN;1062 }1063 } break;1064 case LLM_ARCH_DEEPSEEK2:1065 {1066 bool is_lite = (hparams.n_layer == 27);1067 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1068 ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);1069 if (!is_lite) {1070 ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);1071 }1072 ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);1073 ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);1074 ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);1075 ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);1076 ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);1077 ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);1078 if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {1079 // for compatibility with existing DeepSeek V2 and V2.5 GGUFs1080 // that have no expert_gating_func model parameter set1081 hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX;1082 }1083 ml.get_key(LLM_KV_ROPE_SCALING_YARN_LOG_MUL, hparams.rope_yarn_log_mul);1084 1085 switch (hparams.n_layer) {1086 case 27: type = LLM_TYPE_16B; break;1087 case 60: type = LLM_TYPE_236B; break;1088 case 61: type = LLM_TYPE_671B; break;1089 default: type = LLM_TYPE_UNKNOWN;1090 }1091 } break;1092 case LLM_ARCH_CHATGLM:1093 {1094 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1095 switch (hparams.n_layer) {1096 case 28: {1097 if (hparams.n_head(0) == 16) {1098 type = LLM_TYPE_1_5B;1099 } else {1100 type = LLM_TYPE_6B;1101 }1102 } break;1103 case 40: {1104 if (hparams.n_head(0) == 24) {1105 type = LLM_TYPE_4B;1106 } else {1107 type = LLM_TYPE_9B;1108 }1109 } break;1110 default: type = LLM_TYPE_UNKNOWN;1111 }1112 } break;1113 case LLM_ARCH_BITNET:1114 {1115 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1116 1117 switch (hparams.n_layer) {1118 case 26: type = LLM_TYPE_3B; break;1119 default: type = LLM_TYPE_UNKNOWN;1120 }1121 } break;1122 case LLM_ARCH_T5:1123 {1124 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1125 ml.get_key(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);1126 1127 uint32_t dec_start_token_id;1128 if (ml.get_key(LLM_KV_DECODER_START_TOKEN_ID, dec_start_token_id, false)) {1129 hparams.dec_start_token_id = dec_start_token_id;1130 }1131 1132 switch (hparams.n_layer) {1133 case 6: type = LLM_TYPE_60M; break; // t5-small1134 case 8: type = LLM_TYPE_80M; break; // flan-t5-small1135 case 12:1136 switch (hparams.n_ff()) {1137 case 3072: type = LLM_TYPE_220M; break; // t5-base1138 case 2048: type = LLM_TYPE_250M; break; // flan-t5-base1139 default: type = LLM_TYPE_UNKNOWN;1140 } break;1141 case 24:1142 switch (hparams.n_ff()) {1143 case 4096: type = LLM_TYPE_770M; break; // t5-large1144 case 2816: type = LLM_TYPE_780M; break; // flan-t5-large1145 case 16384: type = LLM_TYPE_3B; break; // t5-3b1146 case 5120: type = LLM_TYPE_3B; break; // flan-t5-xl1147 case 65536: type = LLM_TYPE_11B; break; // t5-11b1148 case 10240: type = LLM_TYPE_11B; break; // flan-t5-xxl1149 default: type = LLM_TYPE_UNKNOWN;1150 } break;1151 default: type = LLM_TYPE_UNKNOWN;1152 }1153 } break;1154 case LLM_ARCH_T5ENCODER:1155 {1156 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1157 ml.get_key(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);1158 type = LLM_TYPE_UNKNOWN;1159 } break;1160 case LLM_ARCH_JAIS:1161 {1162 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);1163 ml.get_key(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, hparams.f_max_alibi_bias);1164 1165 switch (hparams.n_layer) {1166 case 24: type = LLM_TYPE_1_3B; break;1167 case 40: type = LLM_TYPE_13B; break;1168 /* TODO: add variants */1169 default: type = LLM_TYPE_UNKNOWN;1170 }1171 } break;1172 case LLM_ARCH_NEMOTRON:1173 {1174 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);1175 switch (hparams.n_layer) {1176 case 32: type = LLM_TYPE_4B; break;1177 default: type = LLM_TYPE_UNKNOWN;1178 }1179 } break;1180 case LLM_ARCH_EXAONE:1181 {1182 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1183 1184 switch (hparams.n_layer) {1185 case 32: type = LLM_TYPE_8B; break;1186 default: type = LLM_TYPE_UNKNOWN;1187 }1188 } break;1189 case LLM_ARCH_RWKV6:1190 case LLM_ARCH_RWKV6QWEN2:1191 {1192 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps, false);1193 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps, false);1194 ml.get_key(LLM_KV_WKV_HEAD_SIZE, hparams.wkv_head_size);1195 ml.get_key(LLM_KV_TIME_MIX_EXTRA_DIM, hparams.time_mix_extra_dim);1196 ml.get_key(LLM_KV_TIME_DECAY_EXTRA_DIM, hparams.time_decay_extra_dim);1197 ml.get_key(LLM_KV_RESCALE_EVERY_N_LAYERS, hparams.rescale_every_n_layers, false);1198 ml.get_key(LLM_KV_TOKEN_SHIFT_COUNT, hparams.token_shift_count, false);1199 1200 switch (hparams.n_layer) {