echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0604
1#include "llama-model.h"2 3#include "llama-arch.h"4#include "llama-ext.h"5#include "llama-hparams.h"6#include "llama-impl.h"7#include "llama-mmap.h"8#include "llama-cparams.h"9#include "llama-model-loader.h"10 11#include "llama-kv-cache.h"12#include "llama-kv-cache-iswa.h"13#include "llama-memory-hybrid.h"14#include "llama-memory-hybrid-iswa.h"15#include "llama-memory-recurrent.h"16 17#include "models/models.h"18 19#include "ggml.h"20#include "ggml-cpp.h"21 22#include <algorithm>23#include <cassert>24#include <cfloat>25#include <cstdint>26#include <cstring>27#include <cmath>28#include <functional>29#include <map>30#include <numeric>31#include <regex>32#include <sstream>33#include <stdexcept>34#include <string>35#include <vector>36 37struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const struct ggml_tensor * tensor, void * userdata) {38 const llama_meta_device_get_split_state_userdata * ud = (const llama_meta_device_get_split_state_userdata *) userdata;39 const llama_hparams & hparams = ud->model->hparams;40 const std::string tensor_name = tensor->name;41 42 const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight");43 const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight");44 const std::regex pattern_qkv_weight ("blk\\.\\d*\\.attn_qkv.weight");45 const std::regex pattern_q_bias ("blk\\.\\d*\\.attn_q\\.bias");46 const std::regex pattern_kv_bias ("blk\\.\\d*\\.attn_(k|v)\\.bias");47 const std::regex pattern_qkv_bias ("blk\\.\\d*\\.attn_qkv.bias");48 const std::regex pattern_qk_norm ("blk\\.\\d*\\.attn_(q|k)_norm\\.weight");49 const std::regex pattern_kv_cache ("cache_(k|v)_l\\d*");50 const std::regex pattern_attn_sinks ("blk\\.\\d*\\.attn_sinks.weight");51 const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");52 const std::regex pattern_attn_out_bias ("blk\\.\\d*\\.attn_output.bias");53 const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");54 55 const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias");56 const std::regex pattern_ssm_a ("blk\\.\\d*\\.ssm_a");57 const std::regex pattern_ssm_alpha ("blk\\.\\d*\\.ssm_alpha.weight");58 const std::regex pattern_ssm_beta ("blk\\.\\d*\\.ssm_beta.weight");59 const std::regex pattern_ssm_beta_alpha ("blk\\.\\d*\\.ssm_ba.weight");60 const std::regex pattern_r_cache ("cache_r_l\\d*");61 const std::regex pattern_s_cache ("cache_s_l\\d*");62 const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");63 const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");64 65 const std::regex pattern_ffn_up_gate_weight("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.weight");66 const std::regex pattern_ffn_up_gate_bias ("blk\\.\\d*\\.ffn_(up|gate)(_exps)?.bias");67 const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");68 const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight");69 const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias");70 const std::regex pattern_ffn_down_exps_bias("blk\\.\\d*\\.ffn_down_exps.bias");71 72 const std::regex pattern_output_weight("output\\.weight");73 const std::regex pattern_output_bias ("output\\.bias");74 75 struct tensor_config {76 ggml_backend_meta_split_axis axis;77 78 const ggml_tensor * tensor_axis_0;79 80 uint32_t il;81 size_t rotation; // when assigning tensor slices, rotate how the rounding is done for more even allocation82 };83 84 auto get_tensor_config_impl = [&](85 const ggml_backend_meta_split_axis axis, const std::string & suffix = "", const std::string & suffix_fallback = "") -> tensor_config {86 // the layers in a tensor can be inhomogeneous, if the pattern is cleanly divided by the number of GPUs there can be aliasing effects,87 // count only the same type of previous layers to avoid this88 auto get_il_eff = [&](const size_t il){89 size_t ret = 0;90 const bool il_is_recurrent = hparams.is_recurrent(il);91 const bool il_is_swa = hparams.is_swa(il);92 for (size_t il_prev = 0; il_prev < il; il_prev++) {93 ret += hparams.is_recurrent(il_prev) == il_is_recurrent && hparams.is_swa(il_prev) == il_is_swa;94 }95 return ret;96 };97 98 uint32_t il;99 std::string prefix;100 size_t rotation;101 if (tensor_name.substr(0, 4) == "blk.") {102 const size_t length_prefix = tensor_name.find('.', 4);103 GGML_ASSERT(length_prefix != std::string::npos);104 prefix = tensor_name.substr(0, length_prefix + 1);105 il = std::stoull(tensor_name.substr(4, length_prefix));106 rotation = get_il_eff(il) % ud->n_devices;107 } else if (tensor_name.substr(0, 6) == "cache_") {108 const size_t layer_index_start = tensor_name.find("_l", 6);109 GGML_ASSERT(layer_index_start != std::string::npos);110 il = std::stoull(tensor_name.substr(layer_index_start + 2));111 prefix = "blk." + std::to_string(il) + ".";112 rotation = get_il_eff(il) % ud->n_devices;113 } else {114 il = 0;115 rotation = hparams.n_layer % ud->n_devices;116 }117 const ggml_tensor * tensor_axis_0 = suffix.empty() ? tensor : ud->model->get_tensor((prefix + suffix).c_str());118 if (tensor_axis_0 == nullptr) {119 GGML_ASSERT(!suffix_fallback.empty());120 tensor_axis_0 = ud->model->get_tensor((prefix + suffix_fallback).c_str());121 }122 GGML_ASSERT(tensor_axis_0 != nullptr);123 return {axis, tensor_axis_0, il, rotation};124 };125 126 auto get_tensor_config = [&]() -> tensor_config {127 // standard attention128 if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_kv_weight)) {129 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");130 }131 if (std::regex_match(tensor_name, pattern_q_bias) || std::regex_match(tensor_name, pattern_kv_bias)) {132 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight");133 }134 if (std::regex_match(tensor_name, pattern_qkv_weight)) {135 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1);136 }137 if ( std::regex_match(tensor_name, pattern_qkv_bias)) {138 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);139 }140 if (std::regex_match(tensor_name, pattern_qk_norm)) {141 return get_tensor_config_impl(tensor->ne[1] == 1 ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");142 }143 if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks)) {144 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight");145 }146 if (std::regex_match(tensor_name, pattern_attn_out_weight)) {147 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);148 }149 if (std::regex_match(tensor_name, pattern_attn_out_bias)) {150 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);151 }152 153 if (std::regex_match(tensor_name, pattern_attn_gate_weight)) {154 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1);155 }156 if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a)) {157 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");158 }159 if (std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta) ||160 std::regex_match(tensor_name, pattern_ssm_beta_alpha)) {161 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");162 }163 if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) {164 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");165 }166 if (std::regex_match(tensor_name, pattern_ssm_conv1d)) {167 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");168 }169 if (std::regex_match(tensor_name, pattern_ssm_out_weight)) {170 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);171 }172 173 // FFN174 if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight)) {175 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight");176 }177 if (std::regex_match(tensor_name, pattern_ffn_up_gate_bias)) {178 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight");179 }180 if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {181 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight");182 }183 if (std::regex_match(tensor_name, pattern_ffn_down_weight)) {184 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight");185 }186 if (std::regex_match(tensor_name, pattern_ffn_down_bias)) {187 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);188 }189 if (std::regex_match(tensor_name, pattern_ffn_down_exps_bias)) {190 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_PARTIAL);191 }192 193 // output194 if (std::regex_match(tensor_name, pattern_output_weight)) {195 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1);196 }197 if (std::regex_match(tensor_name, pattern_output_bias)) {198 const ggml_tensor * output_weight = ud->model->get_tensor("output.weight");199 GGML_ASSERT(output_weight != nullptr);200 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);201 }202 203 // everything else204 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);205 };206 207 auto get_split_segments = [&](int axis, uint32_t il) -> std::vector<int64_t> {208 if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE) {209 const int64_t head_k_dim = hparams.ssm_d_state;210 const int64_t head_v_dim = hparams.ssm_d_state;211 const int64_t n_k_heads = hparams.ssm_n_group;212 const int64_t n_v_heads = hparams.ssm_dt_rank;213 const int64_t key_dim = head_k_dim * n_k_heads;214 const int64_t value_dim = head_v_dim * n_v_heads;215 216 // both Qwen 3 Next and Qwen 3.5 support n_v_heads > n_k_heads but the broadcasting pattern is different:217 // - Qwen 3 Next: [k0_v0, k0_v1, k1_v2, k1_v3] (this is the default split pattern)218 // - Qwen 3.5: [k0_v0, k1_v1, k0_v2, k1_v3] (needs segmenting of V on the scale of K to get the correct pattern)219 if (ud->model->arch == LLM_ARCH_QWEN3NEXT) {220 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_ssm_conv1d)) {221 GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);222 return {key_dim, key_dim, value_dim};223 }224 } else {225 const int64_t head_ratio = n_v_heads / n_k_heads;226 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_ssm_conv1d)) {227 GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);228 return std::vector<int64_t>(2 + head_ratio, key_dim);229 }230 if (std::regex_match(tensor_name, pattern_attn_gate_weight) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {231 return std::vector<int64_t>(head_ratio, key_dim);232 }233 if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||234 std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) {235 return std::vector<int64_t>(head_ratio, n_k_heads);236 }237 if (std::regex_match(tensor_name, pattern_r_cache)) {238 return std::vector<int64_t>(2 + head_ratio, key_dim * (hparams.ssm_d_conv - 1));239 }240 if (std::regex_match(tensor_name, pattern_s_cache)) {241 return std::vector<int64_t>(head_ratio, n_k_heads * head_v_dim * head_v_dim);242 }243 }244 245 // the FFN is the same for Qwen 3 Next and Qwen 3.5:246 if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {247 const int64_t n_ff_exp = hparams.n_ff_exp;248 GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);249 return {n_ff_exp, n_ff_exp};250 }251 return {tensor->ne[axis]};252 }253 254 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {255 const int64_t n_embd = hparams.n_embd;256 const int64_t n_embd_gqa = hparams.n_embd_v_gqa(il);257 GGML_ASSERT(hparams.n_embd_k_gqa() == n_embd_gqa);258 GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa);259 return {n_embd, n_embd_gqa, n_embd_gqa};260 }261 if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {262 const int64_t n_ff_exp = hparams.n_ff_exp;263 GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);264 return {n_ff_exp, n_ff_exp};265 }266 return {tensor->ne[axis]};267 };268 269 auto get_split_granularity = [&](int64_t blck_size, uint32_t il, const std::vector<int64_t> & segments) -> std::vector<int64_t> {270 if (hparams.is_recurrent(il)) {271 // linear attention272 const int64_t head_dim = hparams.ssm_d_state;273 const int64_t granularity_qkv = std::lcm(blck_size, head_dim);274 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_attn_gate_weight) ||275 std::regex_match(tensor_name, pattern_ssm_conv1d) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {276 return std::vector<int64_t>(segments.size(), granularity_qkv);277 }278 if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||279 std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) {280 return std::vector<int64_t>(segments.size(), granularity_qkv / head_dim);281 }282 if (std::regex_match(tensor_name, pattern_ssm_beta_alpha)) {283 return std::vector<int64_t>(segments.size(), 2 * (granularity_qkv / head_dim));284 }285 if (std::regex_match(tensor_name, pattern_r_cache)) {286 return std::vector<int64_t>(segments.size(), granularity_qkv * (hparams.ssm_d_conv - 1));287 }288 if (std::regex_match(tensor_name, pattern_s_cache)) {289 return std::vector<int64_t>(segments.size(), granularity_qkv * head_dim);290 }291 } else {292 // regular attention293 const uint32_t n_gqa = hparams.n_gqa(il);294 const uint32_t n_embd_q = n_gqa * hparams.n_embd_head_k(il);295 if (std::regex_match(tensor_name, pattern_attn_sinks)) {296 GGML_ASSERT(segments.size() == 1);297 return {std::lcm(n_embd_q, blck_size)/n_embd_q * n_gqa};298 }299 300 const int64_t granularity_q = std::lcm(n_embd_q, blck_size);301 if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_q_bias)) {302 GGML_ASSERT(segments.size() == 1);303 // some models have Q gate tensors, for those cases the granularity needs to be doubled:304 if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE) {305 return {std::lcm(2*n_embd_q, blck_size)};306 }307 return {granularity_q};308 }309 if (std::regex_match(tensor_name, pattern_attn_out_weight)) {310 GGML_ASSERT(segments.size() == 1);311 return {granularity_q};312 }313 314 const int64_t granularity_kv = granularity_q / n_gqa;315 if (std::regex_match(tensor_name, pattern_kv_weight) ||316 std::regex_match(tensor_name, pattern_kv_bias) ||317 std::regex_match(tensor_name, pattern_kv_cache)) {318 GGML_ASSERT(segments.size() == 1);319 return {granularity_kv};320 }321 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {322 GGML_ASSERT(segments.size() == 3);323 return {granularity_q, granularity_kv, granularity_kv};324 }325 }326 327 // FFN328 if (std::regex_match(tensor_name, pattern_ffn_up_gate_weight) || std::regex_match(tensor_name, pattern_ffn_up_gate_bias) ||329 std::regex_match(tensor_name, pattern_ffn_gate_up_weight) || std::regex_match(tensor_name, pattern_ffn_down_weight)) {330 GGML_ASSERT(segments.size() <= 2);331 return std::vector<int64_t>(segments.size(), blck_size);332 }333 334 // everything else335 GGML_ASSERT(segments.size() == 1);336 return {1};337 };338 339 ggml_backend_meta_split_state split_state;340 memset(&split_state, 0, sizeof(split_state));341 tensor_config tc = get_tensor_config();342 split_state.axis = tc.axis;343 if (split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS) {344 const int64_t ne_full = tensor->ne[split_state.axis];345 const int64_t blck_size = ggml_blck_size(tc.tensor_axis_0->type);346 const float * tensor_split = ud->model->tensor_split();347 std::vector<float> tensor_split_scan;348 tensor_split_scan.reserve(ud->n_devices);349 for (size_t j = 0; j < ud->n_devices; j++) {350 tensor_split_scan.push_back(tensor_split == nullptr ? 0.0f : tensor_split[(j + tc.rotation) % ud->n_devices]);351 if (j > 0) {352 tensor_split_scan[j] += tensor_split_scan[j - 1];353 }354 }355 const std::vector<int64_t> segments = get_split_segments(split_state.axis, tc.il);356 const std::vector<int64_t> granularity = get_split_granularity(blck_size, tc.il, segments);357 for (size_t is = 0; is < segments.size(); is++) {358 const int64_t ne_s = segments[is];359 const int64_t g_s = granularity[is];360 GGML_ASSERT(ne_full % g_s == 0);361 int64_t low = 0;362 size_t j = 0;363 for (; j < ud->n_devices - 1; j++) {364 int64_t high = tensor_split_scan.back() == 0.0f ?365 ne_s * (j+1)/ud->n_devices : ne_s * tensor_split_scan[j]/tensor_split_scan.back();366 if (high % g_s != 0) {367 high -= high % g_s;368 }369 split_state.ne[is*ud->n_devices + (j + tc.rotation) % ud->n_devices] = high - low;370 low = high;371 }372 split_state.ne[is*ud->n_devices + (j + tc.rotation) % ud->n_devices] = ne_s - low;373 }374 split_state.n_segments = segments.size();375 } else {376 memset(split_state.ne, 0, sizeof(split_state.ne));377 split_state.n_segments = 1;378 }379 return split_state;380 GGML_UNUSED(userdata);381}382 383const char * llm_type_name(llm_type type) {384 switch (type) {385 case LLM_TYPE_14M: return "14M";386 case LLM_TYPE_17M: return "17M";387 case LLM_TYPE_22M: return "22M";388 case LLM_TYPE_33M: return "33M";389 case LLM_TYPE_47M: return "47M";390 case LLM_TYPE_60M: return "60M";391 case LLM_TYPE_70M: return "70M";392 case LLM_TYPE_80M: return "80M";393 case LLM_TYPE_109M: return "109M";394 case LLM_TYPE_137M: return "137M";395 case LLM_TYPE_140M: return "140M";396 case LLM_TYPE_149M: return "149M";397 case LLM_TYPE_160M: return "160M";398 case LLM_TYPE_190M: return "190M";399 case LLM_TYPE_220M: return "220M";400 case LLM_TYPE_250M: return "250M";401 case LLM_TYPE_256M: return "256M";402 case LLM_TYPE_270M: return "270M";403 case LLM_TYPE_335M: return "335M";404 case LLM_TYPE_350M: return "350M";405 case LLM_TYPE_360M: return "360M";406 case LLM_TYPE_395M: return "395M";407 case LLM_TYPE_410M: return "410M";408 case LLM_TYPE_450M: return "450M";409 case LLM_TYPE_475M: return "475M";410 case LLM_TYPE_558M: return "558M";411 case LLM_TYPE_700M: return "700M";412 case LLM_TYPE_770M: return "770M";413 case LLM_TYPE_780M: return "780M";414 case LLM_TYPE_950M: return "950M";415 case LLM_TYPE_0_3B: return "0.3B";416 case LLM_TYPE_0_5B: return "0.5B";417 case LLM_TYPE_0_6B: return "0.6B";418 case LLM_TYPE_0_8B: return "0.8B";419 case LLM_TYPE_1B: return "1B";420 case LLM_TYPE_1_2B: return "1.2B";421 case LLM_TYPE_1_3B: return "1.3B";422 case LLM_TYPE_1_4B: return "1.4B";423 case LLM_TYPE_1_5B: return "1.5B";424 case LLM_TYPE_1_6B: return "1.6B";425 case LLM_TYPE_1_7B: return "1.7B";426 case LLM_TYPE_1_8B: return "1.8B";427 case LLM_TYPE_2B: return "2B";428 case LLM_TYPE_2_6B: return "2.6B";429 case LLM_TYPE_2_8B: return "2.8B";430 case LLM_TYPE_2_9B: return "2.9B";431 case LLM_TYPE_3B: return "3B";432 case LLM_TYPE_4B: return "4B";433 case LLM_TYPE_6B: return "6B";434 case LLM_TYPE_6_9B: return "6.9B";435 case LLM_TYPE_7B: return "7B";436 case LLM_TYPE_8B: return "8B";437 case LLM_TYPE_9B: return "9B";438 case LLM_TYPE_11B: return "11B";439 case LLM_TYPE_12B: return "12B";440 case LLM_TYPE_13B: return "13B";441 case LLM_TYPE_14B: return "14B";442 case LLM_TYPE_15B: return "15B";443 case LLM_TYPE_16B: return "16B";444 case LLM_TYPE_20B: return "20B";445 case LLM_TYPE_26B: return "26B";446 case LLM_TYPE_27B: return "27B";447 case LLM_TYPE_30B: return "30B";448 case LLM_TYPE_31B: return "31B";449 case LLM_TYPE_32B: return "32B";450 case LLM_TYPE_34B: return "34B";451 case LLM_TYPE_35B: return "35B";452 case LLM_TYPE_36B: return "36B";453 case LLM_TYPE_40B: return "40B";454 case LLM_TYPE_65B: return "65B";455 case LLM_TYPE_70B: return "70B";456 case LLM_TYPE_120B: return "120B";457 case LLM_TYPE_142B: return "142B";458 case LLM_TYPE_236B: return "236B";459 case LLM_TYPE_290B: return "290B";460 case LLM_TYPE_314B: return "314B";461 case LLM_TYPE_405B: return "405B";462 case LLM_TYPE_671B: return "671B";463 case LLM_TYPE_SMALL: return "0.1B";464 case LLM_TYPE_MEDIUM: return "0.4B";465 case LLM_TYPE_LARGE: return "0.8B";466 case LLM_TYPE_XL: return "1.5B";467 case LLM_TYPE_A1_7B: return "A1.7B";468 case LLM_TYPE_A2_7B: return "A2.7B";469 case LLM_TYPE_8x7B: return "8x7B";470 case LLM_TYPE_8x22B: return "8x22B";471 case LLM_TYPE_16x12B: return "16x12B";472 case LLM_TYPE_16x3_8B: return "16x3.8B";473 case LLM_TYPE_10B_128x3_66B: return "10B+128x3.66B";474 case LLM_TYPE_57B_A14B: return "57B.A14B";475 case LLM_TYPE_17B_16E: return "17Bx16E (Scout)";476 case LLM_TYPE_17B_128E: return "17Bx128E (Maverick)";477 case LLM_TYPE_A13B: return "A13B";478 case LLM_TYPE_7B_A1B: return "7B.A1B";479 case LLM_TYPE_8B_A1B: return "8B.A1B";480 case LLM_TYPE_16B_A1B: return "16B.A1B";481 case LLM_TYPE_21B_A3B: return "21B.A3B";482 case LLM_TYPE_24B_A2B: return "24B.A2B";483 case LLM_TYPE_26B_A4B: return "26B.A4B";484 case LLM_TYPE_30B_A3B: return "30B.A3B";485 case LLM_TYPE_31B_A3_5B: return "31B.A3.5B";486 case LLM_TYPE_35B_A3B: return "35B.A3B";487 case LLM_TYPE_48B_A3B: return "48B.A3B";488 case LLM_TYPE_80B_A3B: return "80B.A3B";489 case LLM_TYPE_100B_A6B: return "100B.A6B";490 case LLM_TYPE_102B_A12B: return "102B.A12B";491 case LLM_TYPE_106B_A12B: return "106B.A12B";492 case LLM_TYPE_120B_A12B: return "120B.A12B";493 case LLM_TYPE_122B_A10B: return "122B.A10B";494 case LLM_TYPE_196B_A11B: return "196B.A11B";495 case LLM_TYPE_230B_A10B: return "230B.A10B";496 case LLM_TYPE_235B_A22B: return "235B.A22B";497 case LLM_TYPE_300B_A47B: return "300B.A47B";498 case LLM_TYPE_310B_A15B: return "310B.A15B";499 case LLM_TYPE_355B_A32B: return "355B.A32B";500 case LLM_TYPE_397B_A17B: return "397B.A17B";501 case LLM_TYPE_744B_A40B: return "744B.A40B";502 case LLM_TYPE_E2B: return "E2B";503 case LLM_TYPE_E4B: return "E4B";504 default: return "?B";505 }506}507 508static const char * llama_expert_gating_func_name(llama_expert_gating_func_type type) {509 switch (type) {510 case LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX: return "softmax";511 case LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID: return "sigmoid";512 default: return "unknown";513 }514}515 516static const std::map<llama_rope_scaling_type, const char *> LLAMA_ROPE_SCALING_TYPES = {517 { LLAMA_ROPE_SCALING_TYPE_NONE, "none" },518 { LLAMA_ROPE_SCALING_TYPE_LINEAR, "linear" },519 { LLAMA_ROPE_SCALING_TYPE_YARN, "yarn" },520 { LLAMA_ROPE_SCALING_TYPE_LONGROPE, "longrope" },521};522 523std::string llama_rope_scaling_type_name(llama_rope_scaling_type rope_scaling_type) {524 return LLAMA_ROPE_SCALING_TYPES.at(rope_scaling_type);525}526 527static llama_rope_scaling_type llama_rope_scaling_type_from_string(const std::string & name) {528 for (const auto & kv : LLAMA_ROPE_SCALING_TYPES) {529 if (kv.second == name) {530 return (llama_rope_scaling_type) kv.first;531 }532 }533 534 return LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;535}536 537// CPU: ACCEL -> GPU host -> CPU extra -> CPU538static buft_list_t make_cpu_buft_list(const std::vector<llama_device> & devices, bool use_extra_bufts, bool no_host) {539 buft_list_t buft_list;540 541 // add ACCEL buffer types542 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {543 ggml_backend_dev_t dev = ggml_backend_dev_get(i);544 if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {545 auto * buft = ggml_backend_dev_buffer_type(dev);546 // skip547 if (buft != ggml_backend_cpu_buffer_type()) {548 buft_list.emplace_back(dev, buft);549 }550 }551 }552 553 // add a host buffer type554 // storing the tensors in a host buffer is useful when the processing of large batches555 // is offloaded to a GPU device, since it reduces the time spent on data transfers556 // generally, this will be done using the first device in the list557 // a better approach would be to handle this on a weight-by-weight basis using the offload_op558 // function of the device to determine if it would benefit from being stored in a host buffer559 if (!no_host) {560 for (const auto & dev : devices) {561 ggml_backend_buffer_type_t buft = ggml_backend_dev_host_buffer_type(dev.dev);562 if (buft) {563 buft_list.emplace_back(dev.dev, buft);564 break;565 }566 }567 }568 569 // add extra buffer types570 if (use_extra_bufts) {571 auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);572 if (cpu_dev == nullptr) {573 throw std::runtime_error(format("%s: no CPU backend found", __func__));574 }575 576 auto * cpu_reg = ggml_backend_dev_backend_reg(cpu_dev);577 auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)578 ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_dev_get_extra_bufts");579 if (ggml_backend_dev_get_extra_bufts_fn) {580 ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(cpu_dev);581 while (extra_bufts && *extra_bufts) {582 buft_list.emplace_back(cpu_dev, *extra_bufts);583 ++extra_bufts;584 }585 }586 }587 588 // add the CPU buffer type589 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {590 ggml_backend_dev_t dev = ggml_backend_dev_get(i);591 if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {592 buft_list.emplace_back(dev, ggml_backend_dev_buffer_type(dev));593 }594 }595 596 return buft_list;597}598 599// GPU: split if LLAMA_SPLIT_MODE_ROW -> GPU600static buft_list_t make_gpu_buft_list(ggml_backend_dev_t dev, llama_split_mode split_mode, const float * tensor_split) {601 buft_list_t buft_list;602 603 // add the device split buffer type if requested and available604 if (split_mode == LLAMA_SPLIT_MODE_ROW) {605 ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);606 auto ggml_backend_split_buffer_type_fn = (ggml_backend_split_buffer_type_t)607 ggml_backend_reg_get_proc_address(reg, "ggml_backend_split_buffer_type");608 if (ggml_backend_split_buffer_type_fn) {609 size_t dev_index = [&]() {610 auto * reg = ggml_backend_dev_backend_reg(dev);611 for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); ++i) {612 if (ggml_backend_reg_dev_get(reg, i) == dev) {613 return i;614 }615 }616 throw std::runtime_error(format("device %s not found in its backend reg", ggml_backend_dev_name(dev)));617 }();618 auto * buft = ggml_backend_split_buffer_type_fn(dev_index, tensor_split);619 if (buft != nullptr) {620 buft_list.emplace_back(dev, buft);621 }622 }623 }624 625 // add the device default buffer type626 buft_list.emplace_back(dev, ggml_backend_dev_buffer_type(dev));627 628 // add the device extra buffer type (if any)629 ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);630 if (reg) {631 auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)632 ggml_backend_reg_get_proc_address(reg, "ggml_backend_dev_get_extra_bufts");633 634 if (ggml_backend_dev_get_extra_bufts_fn) {635 ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(dev);636 while (extra_bufts && *extra_bufts) {637 buft_list.emplace_back(dev, *extra_bufts);638 ++extra_bufts;639 }640 }641 }642 643 return buft_list;644}645 646struct llama_model::impl {647 impl() = default;648 ~impl() = default;649 650 uint64_t n_elements = 0;651 652 size_t n_bytes = 0;653 654 std::string desc_str;655 656 // model memory mapped files657 llama_mmaps mappings;658 659 // objects representing data potentially being locked in memory660 llama_mlocks mlock_bufs;661 llama_mlocks mlock_mmaps;662 663 // contexts where the model tensors metadata is stored as well as the corresponding buffers:664 std::vector<std::pair<ggml_context_ptr, std::vector<ggml_backend_buffer_ptr>>> ctxs_bufs;665 666 buft_list_t cpu_buft_list;667 std::map<ggml_backend_dev_t, buft_list_t> gpu_buft_list;668 669 struct layer_dev {670 ggml_backend_dev_t dev;671 buft_list_t * buft_list;672 };673 674 layer_dev dev_input = {};675 layer_dev dev_output = {};676 std::vector<layer_dev> dev_layer;677 678 bool has_tensor_overrides;679};680 681llama_model::llama_model(const llama_model_params & params) : params(params), pimpl(std::make_unique<impl>()) {682 pimpl->has_tensor_overrides = params.tensor_buft_overrides && params.tensor_buft_overrides[0].pattern;683}684 685llama_model::~llama_model() {686 for (auto * lora : loras) {687 delete lora;688 }689}690 691void llama_model::load_stats(llama_model_loader & ml) {692 pimpl->n_elements = ml.n_elements;693 pimpl->n_bytes = ml.n_bytes;694}695 696void llama_model::load_arch(llama_model_loader & ml) {697 arch = ml.get_arch();698 if (arch == LLM_ARCH_UNKNOWN) {699 throw std::runtime_error("unknown model architecture: '" + ml.get_arch_name() + "'");700 }701 if (!devices.empty() && devices[0].is_meta && !llm_arch_supports_sm_tensor(arch)) {702 throw std::runtime_error(std::string("LLAMA_SPLIT_MODE_TENSOR not implemented for architecture '") + llm_arch_name(arch) + "'");703 }704}705 706void llama_model::load_hparams(llama_model_loader & ml) {707 const gguf_context * ctx = ml.metadata;708 709 // get metadata as string710 for (int i = 0; i < gguf_get_n_kv(ctx); i++) {711 gguf_type type = gguf_get_kv_type(ctx, i);712 if (type == GGUF_TYPE_ARRAY) {713 continue;714 }715 const char * name = gguf_get_key(ctx, i);716 const std::string value = gguf_kv_to_str(ctx, i);717 gguf_kv.emplace(name, value);718 }719 720 // get general kv721 ml.get_key(LLM_KV_GENERAL_NAME, name, false);722 723 // everything past this point is not vocab-related724 // for CLIP models, we only need to load tensors, no hparams725 if (hparams.vocab_only || ml.get_arch() == LLM_ARCH_CLIP) {726 return;727 }728 729 ml.get_key(LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train);730 ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd);731 ml.get_key(LLM_KV_EMBEDDING_LENGTH_OUT, hparams.n_embd_out_impl, false);732 ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);733 ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);734 ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer);735 ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);736 ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);737 ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false);738 ml.get_key(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used, false);739 740 if (arch == LLM_ARCH_WAVTOKENIZER_DEC) {741 ml.get_key(LLM_KV_FEATURES_LENGTH, hparams.n_embd);742 ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd_out_impl);743 744 ml.get_key(LLM_KV_POSNET_EMBEDDING_LENGTH, hparams.posnet.n_embd);745 ml.get_key(LLM_KV_POSNET_BLOCK_COUNT, hparams.posnet.n_layer);746 747 ml.get_key(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, hparams.convnext.n_embd);748 ml.get_key(LLM_KV_CONVNEXT_BLOCK_COUNT, hparams.convnext.n_layer);749 }750 751 GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS);752 GGML_ASSERT(hparams.n_expert_used <= hparams.n_expert);753 if (hparams.n_expert > 0) {754 GGML_ASSERT(hparams.n_expert_used > 0);755 GGML_ASSERT(hparams.n_expert_groups < hparams.n_expert);756 if (hparams.n_expert_groups > 1) {757 GGML_ASSERT(hparams.n_expert % hparams.n_expert_groups == 0);758 GGML_ASSERT(hparams.n_group_used > 0);759 GGML_ASSERT(hparams.n_group_used < hparams.n_expert_groups);760 }761 } else {762 GGML_ASSERT(hparams.n_expert_used == 0);763 GGML_ASSERT(hparams.n_expert_groups == 0);764 }765 766 std::fill(hparams.n_head_arr.begin(), hparams.n_head_arr.end(), 0);767 std::fill(hparams.n_head_kv_arr.begin(), hparams.n_head_kv_arr.end(), 0);768 std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0);769 std::fill(770 hparams.recurrent_layer_arr.begin(),771 hparams.recurrent_layer_arr.end(),772 llm_arch_is_recurrent(ml.get_arch()));773 774 std::fill(hparams.rope_sections.begin(), hparams.rope_sections.end(), 0);775 std::fill(hparams.swa_layers.begin(), hparams.swa_layers.end(), 0);776 777 std::fill(hparams.xielu_alpha_n.begin(), hparams.xielu_alpha_n.end(), 0.0f);778 std::fill(hparams.xielu_alpha_p.begin(), hparams.xielu_alpha_p.end(), 0.0f);779 std::fill(hparams.xielu_beta.begin(), hparams.xielu_beta.end(), 0.0f);780 std::fill(hparams.xielu_eps.begin(), hparams.xielu_eps.end(), 0.0f);781 std::fill(hparams.swiglu_clamp_exp.begin(), hparams.swiglu_clamp_exp.end(), 0.0f);782 std::fill(hparams.swiglu_clamp_shexp.begin(), hparams.swiglu_clamp_shexp.end(), 0.0f);783 784 ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer, false);785 ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer, false);786 787 // n_head_kv is optional, default to n_head788 hparams.n_head_kv_arr = hparams.n_head_arr;789 790 ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, hparams.n_layer, false);791 792 bool rope_finetuned = false;793 ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);794 hparams.rope_finetuned = rope_finetuned;795 796 hparams.n_ctx_orig_yarn = hparams.n_ctx_train;797 ml.get_key(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, hparams.n_ctx_orig_yarn, false);798 799 // rope_freq_base (optional)800 hparams.rope_freq_base_train = 10000.0f;801 ml.get_key(LLM_KV_ROPE_FREQ_BASE, hparams.rope_freq_base_train, false);802 803 std::string rope_scaling("linear");804 ml.get_key(LLM_KV_ROPE_SCALING_TYPE, rope_scaling, false);805 hparams.rope_scaling_type_train = llama_rope_scaling_type_from_string(rope_scaling);806 GGML_ASSERT(hparams.rope_scaling_type_train != LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED);807 808 // TODO: Handle SWA metadata similarly when models start implementing it809 // rope_freq_scale (inverse of the kv) is optional810 float ropescale = 0.0f;811 if (!ml.get_key(LLM_KV_ROPE_SCALING_FACTOR, ropescale, false)) {812 // try the old key name813 ml.get_key(LLM_KV_ROPE_SCALE_LINEAR, ropescale, false);814 }815 hparams.rope_freq_scale_train = ropescale == 0.0f ? 1.0f : 1.0f/ropescale;816 817 ml.get_key(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor, false);818 819 // non-transformer models do not have attention heads820 if (hparams.n_head() > 0) {821 // gpt-neox n_rot = rotary_pct * (n_embd / n_head)822 // gpt-j n_rot = rotary_dim823 824 hparams.n_embd_head_k_full = hparams.n_embd / hparams.n_head();825 ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k_full, false);826 827 hparams.n_embd_head_v_full = hparams.n_embd / hparams.n_head();828 ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v_full, false);829 830 // sanity check for n_rot (optional)831 hparams.n_rot_full = hparams.n_embd_head_k_full;832 833 ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT, hparams.n_rot_full, false);834 835 if (arch == LLM_ARCH_LLAMA || arch == LLM_ARCH_DECI || arch == LLM_ARCH_FALCON || arch == LLM_ARCH_LLAMA_EMBED) {836 if (hparams.n_rot_full != hparams.n_embd_head_k_full) {837 throw std::runtime_error(format("invalid n_rot: %u, expected %u", hparams.n_rot_full, hparams.n_embd_head_k_full));838 }839 }840 } else {841 hparams.n_rot_full = 0;842 hparams.n_embd_head_k_full = 0;843 hparams.n_embd_head_v_full = 0;844 }845 846 // head size and n_rot for SWA layers847 {848 hparams.n_embd_head_k_swa = hparams.n_embd_head_k_full;849 hparams.n_embd_head_v_swa = hparams.n_embd_head_v_full;850 ml.get_key(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa, false);851 ml.get_key(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa, false);852 853 hparams.n_rot_swa = hparams.n_rot_full;854 ml.get_key(LLM_KV_ROPE_DIMENSION_COUNT_SWA, hparams.n_rot_swa, false);855 }856 857 // for differentiating model types858 uint32_t n_vocab = 0;859 ml.get_key(LLM_KV_VOCAB_SIZE, n_vocab, false) || ml.get_arr_n(LLM_KV_TOKENIZER_LIST, n_vocab, false);860 861 // for classifier models862 ml.get_arr(LLM_KV_CLASSIFIER_OUTPUT_LABELS, classifier_labels, false);863 if (!classifier_labels.empty()) {864 hparams.n_cls_out = classifier_labels.size();865 }866 867 // arch-specific KVs868 switch (arch) {869 case LLM_ARCH_LLAMA:870 case LLM_ARCH_LLAMA_EMBED:871 {872 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);873 874 if (hparams.n_expert == 8) {875 switch (hparams.n_layer) {876 case 32: type = LLM_TYPE_8x7B; break;877 case 56: type = LLM_TYPE_8x22B; break;878 default: type = LLM_TYPE_UNKNOWN;879 }880 } else {881 switch (hparams.n_layer) {882 case 16: type = LLM_TYPE_1B; break; // Llama 3.2 1B883 case 22: type = LLM_TYPE_1B; break;884 case 26: type = LLM_TYPE_3B; break;885 case 28: type = LLM_TYPE_3B; break; // Llama 3.2 3B886 case 30: type = LLM_TYPE_256M; break; // smoldocling 256M887 // granite uses a vocab with len 49152888 case 32: type = n_vocab == 49152 ? LLM_TYPE_3B : (n_vocab < 40000 ? LLM_TYPE_7B : LLM_TYPE_8B); break;889 case 36: type = LLM_TYPE_8B; break; // granite890 case 40: type = LLM_TYPE_13B; break;891 case 48: type = LLM_TYPE_34B; break;892 case 60: type = LLM_TYPE_30B; break;893 case 80: type = hparams.n_head() == hparams.n_head_kv() ? LLM_TYPE_65B : LLM_TYPE_70B; break;894 default: type = LLM_TYPE_UNKNOWN;895 }896 }897 } break;898 case LLM_ARCH_LLAMA4:899 {900 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);901 ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);902 ml.get_key(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step);903 904 const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);905 if (found_swa && hparams.n_swa == 0) {906 hparams.swa_type = LLAMA_SWA_TYPE_NONE;907 hparams.n_no_rope_layer_step = hparams.n_layer; // always use rope908 } else {909 hparams.swa_type = LLAMA_SWA_TYPE_CHUNKED;910 hparams.n_swa = 8192;911 hparams.n_attn_temp_floor_scale = 8192;912 hparams.f_attn_temp_scale = 0.1f;913 hparams.f_attn_temp_offset = 1.0f;914 uint32_t swa_period = 4; // pattern: 3 chunked - 1 full915 ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);916 hparams.set_swa_pattern(swa_period);917 918 hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;919 hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;920 ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);921 }922 923 switch (hparams.n_expert) {924 case 0: {925 // MobileLLM (no MoE)926 switch (hparams.n_embd) {927 case 2048: type = LLM_TYPE_140M; break;928 case 4096: type = LLM_TYPE_360M; break;929 case 6144: type = LLM_TYPE_950M; break;930 default: type = LLM_TYPE_UNKNOWN;931 }932 } break;933 case 16: type = LLM_TYPE_17B_16E; break;934 case 128: type = LLM_TYPE_17B_128E; break;935 default: type = LLM_TYPE_UNKNOWN;936 }937 938 hparams.use_kq_norm = type != LLM_TYPE_17B_128E;939 } break;940 case LLM_ARCH_ARCEE:941 {942 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);943 944 // Arcee uses the same structure as Llama945 switch (hparams.n_layer) {946 case 36: type = LLM_TYPE_4B; break;947 default: type = LLM_TYPE_UNKNOWN;948 }949 } break;950 case LLM_ARCH_AFMOE:951 {952 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);953 ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false);954 ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);955 ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);956 ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false);957 ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false);958 ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false);959 ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);960 961 // Set up interleaved sliding window attention (ISWA)962 // Pattern: 3 sliding - 1 full (global_attn_every_n_layers = 4)963 if (hparams.n_swa > 0) {964 hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;965 uint32_t swa_period = 4;966 ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);967 hparams.set_swa_pattern(swa_period);968 969 hparams.rope_freq_base_train_swa = hparams.rope_freq_base_train;970 hparams.rope_freq_scale_train_swa = hparams.rope_freq_scale_train;971 ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);972 } else {973 hparams.swa_type = LLAMA_SWA_TYPE_NONE;974 }975 976 // Default to sigmoid if not set977 if (hparams.expert_gating_func == LLAMA_EXPERT_GATING_FUNC_TYPE_NONE) {978 hparams.expert_gating_func = LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID;979 }980 981 switch (hparams.n_layer) {982 case 56: type = LLM_TYPE_6B; break;983 case 32: type = LLM_TYPE_26B; break;984 default: type = LLM_TYPE_UNKNOWN;985 }986 } break;987 case LLM_ARCH_DECI:988 {989 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);990 switch (hparams.n_layer) {991 case 32: type = LLM_TYPE_7B; break;992 case 80: type = LLM_TYPE_70B; break;993 case 162: type = LLM_TYPE_405B; break;994 default: type = LLM_TYPE_UNKNOWN;995 }996 } break;997 case LLM_ARCH_MINICPM:998 {999 // Backward-compatible defaults for older MiniCPM GGUFs1000 hparams.f_embedding_scale = 12.0f;1001 hparams.f_residual_scale = 1.4f / sqrtf(float(hparams.n_layer));1002 hparams.f_logit_scale = hparams.n_embd ? (256.0f / float(hparams.n_embd)) : 1.0f;1003 1004 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1005 1006 // Optional KV reads, override defaults if present in newer GGUF exports1007 ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, /*required=*/false);1008 ml.get_key(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale, /*required=*/false);1009 ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale, /*required=*/false);1010 1011 // MiniCPM uses rope by default, unlike Granite which uses it as a switch1012 hparams.rope_finetuned = true;1013 1014 switch (hparams.n_layer) {1015 case 52: type = LLM_TYPE_1B; break;1016 case 40: type = LLM_TYPE_2B; break;1017 default: type = LLM_TYPE_UNKNOWN;1018 }1019 } break;1020 case LLM_ARCH_MINICPM3:1021 {1022 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1023 ml.get_key(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);1024 ml.get_key(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);1025 1026 switch (hparams.n_layer) {1027 case 62: type = LLM_TYPE_4B; break;1028 default: type = LLM_TYPE_UNKNOWN;1029 }1030 } break;1031 case LLM_ARCH_GROK:1032 {1033 // defaults for old GGUFs1034 hparams.yarn_beta_fast = 8.0f;1035 hparams.f_logit_scale = 0.5773502691896257f;1036 hparams.f_embedding_scale = 78.38367176906169f;1037 hparams.f_attn_out_scale = 0.08838834764831845f;1038 hparams.f_attn_logit_softcapping = 30.0f;1039 hparams.f_router_logit_softcapping = 30.0f;1040 // no final_logit_softcapping in grok-11041 hparams.f_final_logit_softcapping = 0.0f;1042 1043 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1044 ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp, false);1045 ml.get_key(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale, false);1046 ml.get_key(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale, false);1047 ml.get_key(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale, false);1048 ml.get_key(LLM_KV_ATTN_LOGIT_SOFTCAPPING, hparams.f_attn_logit_softcapping, false);1049 ml.get_key(LLM_KV_ROUTER_LOGIT_SOFTCAPPING, hparams.f_router_logit_softcapping, false);1050 ml.get_key(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping, false);1051 1052 ml.get_key(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.attn_temp_length, false);1053 ml.get_key(LLM_KV_ROPE_SCALING_YARN_EXT_FACTOR, hparams.yarn_ext_factor, false);1054 ml.get_key(LLM_KV_ROPE_SCALING_YARN_ATTN_FACTOR, hparams.yarn_attn_factor, false);1055 ml.get_key(LLM_KV_ROPE_SCALING_YARN_BETA_FAST, hparams.yarn_beta_fast, false);1056 ml.get_key(LLM_KV_ROPE_SCALING_YARN_BETA_SLOW, hparams.yarn_beta_slow, false);1057 1058 switch (hparams.n_layer) {1059 case 64: type = LLM_TYPE_314B; break;1060 default: type = LLM_TYPE_UNKNOWN;1061 }1062 } break;1063 case LLM_ARCH_FALCON:1064 {1065 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);1066 1067 switch (hparams.n_layer) {1068 case 32: type = LLM_TYPE_7B; break;1069 case 60: type = LLM_TYPE_40B; break;1070 default: type = LLM_TYPE_UNKNOWN;1071 }1072 } break;1073 case LLM_ARCH_BAICHUAN:1074 {1075 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1076 switch (hparams.n_layer) {1077 case 32: type = LLM_TYPE_7B; break;1078 case 40: type = LLM_TYPE_13B; break;1079 default: type = LLM_TYPE_UNKNOWN;1080 }1081 1082 if (type == LLM_TYPE_13B) {1083 // TODO: become GGUF KV parameter1084 hparams.f_max_alibi_bias = 8.0f;1085 }1086 } break;1087 case LLM_ARCH_STARCODER:1088 {1089 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);1090 switch (hparams.n_layer) {1091 case 24: type = LLM_TYPE_1B; break;1092 case 36: type = LLM_TYPE_3B; break;1093 case 42: type = LLM_TYPE_7B; break;1094 case 40: type = LLM_TYPE_15B; break;1095 default: type = LLM_TYPE_UNKNOWN;1096 }1097 } break;1098 case LLM_ARCH_REFACT:1099 {1100 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1101 switch (hparams.n_layer) {1102 case 32: type = LLM_TYPE_1B; break;1103 default: type = LLM_TYPE_UNKNOWN;1104 }1105 1106 // TODO: become GGUF KV parameter1107 hparams.f_max_alibi_bias = 8.0f;1108 } break;1109 case LLM_ARCH_BERT:1110 {1111 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);1112 1113 switch (hparams.n_layer) {1114 case 3:1115 type = LLM_TYPE_17M; break; // bge-micro1116 case 6:1117 type = LLM_TYPE_22M; break; // MiniLM-L61118 case 12:1119 switch (hparams.n_embd) {1120 case 384: type = LLM_TYPE_33M; break; // MiniLM-L12, bge-small1121 case 768: type = LLM_TYPE_109M; break; // bge-base1122 default: type = LLM_TYPE_UNKNOWN;1123 } break;1124 case 24:1125 type = LLM_TYPE_335M; break; // bge-large1126 default: type = LLM_TYPE_UNKNOWN;1127 }1128 } break;1129 case LLM_ARCH_MODERN_BERT:1130 {1131 const bool found_swa = ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false);1132 if (found_swa && hparams.n_swa > 0) {1133 hparams.swa_type = LLAMA_SWA_TYPE_SYMMETRIC;1134 ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa, false);1135 uint32_t swa_period = 3;1136 ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, swa_period, false);1137 hparams.set_swa_pattern(swa_period, true);1138 } else {1139 hparams.swa_type = LLAMA_SWA_TYPE_NONE;1140 }1141 1142 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);1143 1144 switch (hparams.n_layer) {1145 case 12:1146 type = LLM_TYPE_47M; break; // granite-embedding-small1147 case 22:1148 type = LLM_TYPE_149M; break; // modern-bert-base1149 case 28:1150 type = LLM_TYPE_395M; break; // modern-bert-large1151 default: type = LLM_TYPE_UNKNOWN;1152 }1153 } break;1154 case LLM_ARCH_JINA_BERT_V2:1155 {1156 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);1157 hparams.f_max_alibi_bias = 8.0f;1158 1159 switch (hparams.n_layer) {1160 case 4: type = LLM_TYPE_33M; break; // jina-embeddings-small1161 case 12: type = LLM_TYPE_137M; break; // jina-embeddings-base1162 default: type = LLM_TYPE_UNKNOWN;1163 }1164 } break;1165 case LLM_ARCH_JINA_BERT_V3:1166 {1167 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);1168 1169 switch (hparams.n_layer) {1170 case 24:1171 type = LLM_TYPE_558M; break;1172 default: type = LLM_TYPE_UNKNOWN;1173 }1174 } break;1175 case LLM_ARCH_NOMIC_BERT:1176 case LLM_ARCH_NOMIC_BERT_MOE:1177 {1178 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);1179 ml.get_key(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers, 0);1180 1181 if (hparams.n_layer == 12 && hparams.n_embd == 768) {1182 if (arch == LLM_ARCH_NOMIC_BERT) {1183 type = LLM_TYPE_137M;1184 } else if (arch == LLM_ARCH_NOMIC_BERT_MOE && hparams.moe_every_n_layers == 2) {1185 type = LLM_TYPE_475M;1186 }1187 }1188 } break;1189 case LLM_ARCH_NEO_BERT:1190 {1191 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1192 1193 if (hparams.n_layer == 28) {1194 type = LLM_TYPE_250M;1195 }1196 } break;1197 case LLM_ARCH_EUROBERT:1198 {1199 ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);1200