Brunobkr/llama.cpp_AlgMor24_github
ΩFFFΣLLIa • llama.cpp • AlgMor24 ██████╗ ███████╗███████╗███████╗██╗ ██╗ ██╗ █████╗ ██╔═══██╗██╔════╝██╔════╝██╔════╝██║ ██║ ██║██╔══██╗ ██║ ██║█████╗ █████╗ █████╗ ██║ ██║ ██║███████║ ██║ ██║██╔══╝ ██╔══╝ ██╔══╝ ██║ ██║ ██║██╔══██║ ╚██████╔╝██║ ██║ ███████╗███████╗███████╗██║██║ ██║ ╚═════╝ ╚═╝ ╚═╝ ╚══════╝╚══════╝╚══════╝╚═╝╚═╝ ╚═╝ High-Performance LLM / VLM Inference & Autonomous Agentic Ecosystem… See the full description on the dataset page: https://huggingface.co/datasets/Brunobkr/llama.cpp_AlgMor24_github.
03k
1#include "llama-model.h"2 3#include "llama-arch.h"4#include "llama-ext.h"5#include "llama-hparams.h"6#include "llama-impl.h"7#include "llama-mmap.h"8#include "llama-cparams.h"9#include "llama-model-loader.h"10 11#include "llama-kv-cache.h"12#include "llama-kv-cache-iswa.h"13#include "llama-kv-cache-dsa.h"14#include "llama-kv-cache-msa.h"15#include "llama-kv-cache-dsv4.h"16#include "llama-memory-hybrid.h"17#include "llama-memory-hybrid-iswa.h"18#include "llama-memory-recurrent.h"19 20#include "llama.h"21#include "models/models.h"22 23#include "ggml.h"24#include "ggml-cpp.h"25 26#include <algorithm>27#include <cassert>28#include <cfloat>29#include <cstdint>30#include <cstring>31#include <cmath>32#include <functional>33#include <map>34#include <numeric>35#include <regex>36#include <sstream>37#include <stdexcept>38#include <string>39#include <vector>40 41static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params & params) {42 switch (arch) {43 case LLM_ARCH_CLIP:44 return new llama_model_clip(params);45 case LLM_ARCH_LLAMA:46 return new llama_model_llama(params);47 case LLM_ARCH_LLAMA4:48 return new llama_model_llama4(params);49 case LLM_ARCH_LLAMA_EMBED:50 return new llama_model_llama_embed(params);51 case LLM_ARCH_MAINCODER:52 return new llama_model_maincoder(params);53 case LLM_ARCH_TALKIE:54 return new llama_model_talkie(params);55 case LLM_ARCH_DECI:56 return new llama_model_deci(params);57 case LLM_ARCH_BAICHUAN:58 return new llama_model_baichuan(params);59 case LLM_ARCH_FALCON:60 return new llama_model_falcon(params);61 case LLM_ARCH_GROK:62 return new llama_model_grok(params);63 case LLM_ARCH_STARCODER:64 return new llama_model_starcoder(params);65 case LLM_ARCH_REFACT:66 return new llama_model_refact(params);67 case LLM_ARCH_BERT:68 return new llama_model_bert(params);69 case LLM_ARCH_JINA_BERT_V2:70 return new llama_model_jina_bert_v2(params);71 case LLM_ARCH_JINA_BERT_V3:72 return new llama_model_jina_bert_v3(params);73 case LLM_ARCH_NOMIC_BERT:74 return new llama_model_nomic_bert(params);75 case LLM_ARCH_NOMIC_BERT_MOE:76 return new llama_model_nomic_bert_moe(params);77 case LLM_ARCH_MODERN_BERT:78 return new llama_model_modern_bert(params);79 case LLM_ARCH_NEO_BERT:80 return new llama_model_neo_bert(params);81 case LLM_ARCH_EUROBERT:82 return new llama_model_eurobert(params);83 case LLM_ARCH_BLOOM:84 return new llama_model_bloom(params);85 case LLM_ARCH_MPT:86 return new llama_model_mpt(params);87 case LLM_ARCH_STABLELM:88 return new llama_model_stablelm(params);89 case LLM_ARCH_MELLUM:90 return new llama_model_mellum(params);91 case LLM_ARCH_NANBEIGE:92 return new llama_model_nanbeige(params);93 case LLM_ARCH_QWEN:94 return new llama_model_qwen(params);95 case LLM_ARCH_QWEN2:96 return new llama_model_qwen2(params);97 case LLM_ARCH_DREAM:98 return new llama_model_dream(params);99 case LLM_ARCH_LLADA:100 return new llama_model_llada(params);101 case LLM_ARCH_LLADA_MOE:102 return new llama_model_llada_moe(params);103 case LLM_ARCH_RND1:104 return new llama_model_rnd1(params);105 case LLM_ARCH_QWEN2VL:106 return new llama_model_qwen2vl(params);107 case LLM_ARCH_QWEN2MOE:108 return new llama_model_qwen2moe(params);109 case LLM_ARCH_QWEN3:110 return new llama_model_qwen3(params);111 case LLM_ARCH_QWEN3MOE:112 return new llama_model_qwen3moe(params);113 case LLM_ARCH_QWEN3VL:114 return new llama_model_qwen3vl(params);115 case LLM_ARCH_QWEN3VLMOE:116 return new llama_model_qwen3vlmoe(params);117 case LLM_ARCH_QWEN3TTS:118 return new llama_model_qwen3tts(params);119 case LLM_ARCH_PHI2:120 return new llama_model_phi2(params);121 case LLM_ARCH_PHI3:122 return new llama_model_phi3(params);123 case LLM_ARCH_PHIMOE:124 return new llama_model_phimoe(params);125 case LLM_ARCH_PLAMO:126 return new llama_model_plamo(params);127 case LLM_ARCH_PLAMO2:128 return new llama_model_plamo2(params);129 case LLM_ARCH_PLAMO3:130 return new llama_model_plamo3(params);131 case LLM_ARCH_GPT2:132 return new llama_model_gpt2(params);133 case LLM_ARCH_CODESHELL:134 return new llama_model_codeshell(params);135 case LLM_ARCH_ORION:136 return new llama_model_orion(params);137 case LLM_ARCH_INTERNLM2:138 return new llama_model_internlm2(params);139 case LLM_ARCH_MINICPM3:140 return new llama_model_minicpm3(params);141 case LLM_ARCH_GEMMA:142 return new llama_model_gemma(params);143 case LLM_ARCH_GEMMA2:144 return new llama_model_gemma2(params);145 case LLM_ARCH_GEMMA3:146 return new llama_model_gemma3(params);147 case LLM_ARCH_GEMMA3N:148 return new llama_model_gemma3n(params);149 case LLM_ARCH_GEMMA4:150 return new llama_model_gemma4(params);151 case LLM_ARCH_GEMMA4_ASSISTANT:152 return new llama_model_gemma4_assistant(params);153 case LLM_ARCH_GEMMA_EMBEDDING:154 return new llama_model_gemma_embedding(params);155 case LLM_ARCH_STARCODER2:156 return new llama_model_starcoder2(params);157 case LLM_ARCH_MAMBA:158 return new llama_model_mamba(params);159 case LLM_ARCH_MAMBA2:160 return new llama_model_mamba2(params);161 case LLM_ARCH_JAMBA:162 return new llama_model_jamba(params);163 case LLM_ARCH_XVERSE:164 return new llama_model_xverse(params);165 case LLM_ARCH_COMMAND_R:166 return new llama_model_command_r(params);167 case LLM_ARCH_COHERE2:168 return new llama_model_cohere2(params);169 case LLM_ARCH_COHERE2MOE:170 return new llama_model_cohere2moe(params);171 case LLM_ARCH_DBRX:172 return new llama_model_dbrx(params);173 case LLM_ARCH_OLMO:174 return new llama_model_olmo(params);175 case LLM_ARCH_OLMO2:176 return new llama_model_olmo2(params);177 case LLM_ARCH_OLMOE:178 return new llama_model_olmoe(params);179 case LLM_ARCH_MUSE_GLIMMER:180 return new llama_model_muse_glimmer(params);181 case LLM_ARCH_OPENELM:182 return new llama_model_openelm(params);183 case LLM_ARCH_GPTNEOX:184 return new llama_model_gptneox(params);185 case LLM_ARCH_ARCTIC:186 return new llama_model_arctic(params);187 case LLM_ARCH_DEEPSEEK:188 return new llama_model_deepseek(params);189 case LLM_ARCH_DEEPSEEK2:190 return new llama_model_deepseek2(params);191 case LLM_ARCH_DEEPSEEK2OCR:192 return new llama_model_deepseek2ocr(params);193 case LLM_ARCH_DEEPSEEK32:194 return new llama_model_deepseek32(params);195 case LLM_ARCH_DEEPSEEK4:196 return new llama_model_deepseek4(params);197 case LLM_ARCH_GLM_DSA:198 return new llama_model_glm_dsa(params);199 case LLM_ARCH_MISTRAL4:200 return new llama_model_mistral4(params);201 case LLM_ARCH_CHATGLM:202 return new llama_model_chatglm(params);203 case LLM_ARCH_GLM4:204 return new llama_model_glm4(params);205 case LLM_ARCH_GLM4_MOE:206 return new llama_model_glm4_moe(params);207 case LLM_ARCH_BITNET:208 return new llama_model_bitnet(params);209 case LLM_ARCH_T5:210 return new llama_model_t5(params);211 case LLM_ARCH_T5ENCODER:212 return new llama_model_t5encoder(params);213 case LLM_ARCH_JAIS:214 return new llama_model_jais(params);215 case LLM_ARCH_JAIS2:216 return new llama_model_jais2(params);217 case LLM_ARCH_NEMOTRON:218 return new llama_model_nemotron(params);219 case LLM_ARCH_NEMOTRON_H:220 return new llama_model_nemotron_h(params);221 case LLM_ARCH_NEMOTRON_H_MOE:222 return new llama_model_nemotron_h_moe(params);223 case LLM_ARCH_EXAONE:224 return new llama_model_exaone(params);225 case LLM_ARCH_EXAONE4:226 return new llama_model_exaone4(params);227 case LLM_ARCH_EXAONE_MOE:228 return new llama_model_exaone_moe(params);229 case LLM_ARCH_RWKV6:230 return new llama_model_rwkv6(params);231 case LLM_ARCH_RWKV6QWEN2:232 return new llama_model_rwkv6qwen2(params);233 case LLM_ARCH_RWKV7:234 return new llama_model_rwkv7(params);235 case LLM_ARCH_ARWKV7:236 return new llama_model_arwkv7(params);237 case LLM_ARCH_GRANITE:238 return new llama_model_granite(params);239 case LLM_ARCH_GRANITE_MOE:240 return new llama_model_granite_moe(params);241 case LLM_ARCH_GRANITE_SWITCH:242 return new llama_model_granite_switch(params);243 case LLM_ARCH_MINICPM:244 return new llama_model_minicpm(params);245 case LLM_ARCH_GRANITE_HYBRID:246 return new llama_model_granite_hybrid(params);247 case LLM_ARCH_CHAMELEON:248 return new llama_model_chameleon(params);249 case LLM_ARCH_WAVTOKENIZER_DEC:250 return new llama_model_wavtokenizer_dec(params);251 case LLM_ARCH_PLM:252 return new llama_model_plm(params);253 case LLM_ARCH_BAILINGMOE:254 return new llama_model_bailingmoe(params);255 case LLM_ARCH_BAILINGMOE2:256 return new llama_model_bailingmoe2(params);257 case LLM_ARCH_SEED_OSS:258 return new llama_model_seed_oss(params);259 case LLM_ARCH_DOTS1:260 return new llama_model_dots1(params);261 case LLM_ARCH_ARCEE:262 return new llama_model_arcee(params);263 case LLM_ARCH_AFMOE:264 return new llama_model_afmoe(params);265 case LLM_ARCH_LAGUNA:266 return new llama_model_laguna(params);267 case LLM_ARCH_ERNIE4_5:268 return new llama_model_ernie4_5(params);269 case LLM_ARCH_ERNIE4_5_MOE:270 return new llama_model_ernie4_5_moe(params);271 case LLM_ARCH_PADDLEOCR:272 return new llama_model_paddleocr(params);273 case LLM_ARCH_HUNYUAN_MOE:274 return new llama_model_hunyuan_moe(params);275 case LLM_ARCH_HUNYUAN_VL:276 return new llama_model_hunyuan_vl(params);277 case LLM_ARCH_HUNYUAN_DENSE:278 return new llama_model_hunyuan_dense(params);279 case LLM_ARCH_HY_V3:280 return new llama_model_hy_v3(params);281 case LLM_ARCH_SMOLLM3:282 return new llama_model_smollm3(params);283 case LLM_ARCH_OPENAI_MOE:284 return new llama_model_openai_moe(params);285 case LLM_ARCH_FALCON_H1:286 return new llama_model_falcon_h1(params);287 case LLM_ARCH_LFM2:288 return new llama_model_lfm2(params);289 case LLM_ARCH_LFM2MOE:290 return new llama_model_lfm2moe(params);291 case LLM_ARCH_SMALLTHINKER:292 return new llama_model_smallthinker(params);293 case LLM_ARCH_GROVEMOE:294 return new llama_model_grovemoe(params);295 case LLM_ARCH_APERTUS:296 return new llama_model_apertus(params);297 case LLM_ARCH_MINIMAX_M2:298 return new llama_model_minimax_m2(params);299 case LLM_ARCH_MINIMAX_M3:300 return new llama_model_minimax_m3(params);301 case LLM_ARCH_COGVLM:302 return new llama_model_cogvlm(params);303 case LLM_ARCH_PANGU_EMBED:304 return new llama_model_pangu_embed(params);305 case LLM_ARCH_QWEN3NEXT:306 return new llama_model_qwen3next(params);307 case LLM_ARCH_QWEN35:308 return new llama_model_qwen35(params);309 case LLM_ARCH_QWEN35MOE:310 return new llama_model_qwen35moe(params);311 case LLM_ARCH_MISTRAL3:312 return new llama_model_mistral3(params);313 case LLM_ARCH_EAGLE3:314 return new llama_model_eagle3(params);315 case LLM_ARCH_DFLASH:316 return new llama_model_dflash(params);317 case LLM_ARCH_MIMO2:318 return new llama_model_mimo2(params);319 case LLM_ARCH_KIMI_LINEAR:320 return new llama_model_kimi_linear(params);321 case LLM_ARCH_STEP35:322 return new llama_model_step35(params);323 default:324 throw std::runtime_error(std::string("unsupported model architecture: '") + llm_arch_name(arch) + "'");325 }326 327}328 329llama_model * llama_model_create(llm_arch arch, const llama_model_params & params) {330 llama_model * model = llama_model_mapping(arch, params);331 332 if (model != nullptr) {333 model->arch = arch;334 if (params.split_mode == LLAMA_SPLIT_MODE_TENSOR && !llm_arch_supports_sm_tensor(arch)) {335 throw std::runtime_error(std::string("LLAMA_SPLIT_MODE_TENSOR not implemented for architecture '") + llm_arch_name(arch) + "'");336 }337 }338 339 return model;340}341 342llama_model * llama_model_create(llama_model_loader & ml, const llama_model_params & params) {343 llm_arch arch = ml.get_arch();344 if (arch == LLM_ARCH_UNKNOWN) {345 throw std::runtime_error("unknown model architecture: '" + ml.get_arch_name() + "'");346 }347 348 return llama_model_create(arch, params);349}350 351struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const struct ggml_tensor * tensor, void * userdata) {352 const llama_meta_device_get_split_state_userdata * ud = (const llama_meta_device_get_split_state_userdata *) userdata;353 const llama_hparams & hparams = ud->model->hparams;354 const std::string tensor_name = tensor->name;355 356 static const std::regex pattern_q_weight ("blk\\.\\d*\\.attn_q.weight");357 static const std::regex pattern_kv_weight ("blk\\.\\d*\\.attn_(k|v).weight");358 static const std::regex pattern_qkv_weight ("blk\\.\\d*\\.attn_qkv.weight");359 static const std::regex pattern_q_bias ("blk\\.\\d*\\.attn_q\\.bias");360 static const std::regex pattern_kv_bias ("blk\\.\\d*\\.attn_(k|v)\\.bias");361 static const std::regex pattern_qkv_bias ("blk\\.\\d*\\.attn_qkv.bias");362 static const std::regex pattern_qk_norm ("blk\\.\\d*\\.attn_(q|k)_norm\\.weight");363 static const std::regex pattern_kv_cache ("cache_(k|v)_l\\d*");364 static const std::regex pattern_attn_sinks ("blk\\.\\d*\\.attn_sinks.weight");365 static const std::regex pattern_attn_out_weight ("blk\\.\\d*\\.attn_output.weight");366 static const std::regex pattern_attn_out_bias ("blk\\.\\d*\\.attn_output.bias");367 static const std::regex pattern_attn_gate_weight("blk\\.\\d*\\.attn_gate.weight");368 369 static const std::regex pattern_ssm_dt ("blk\\.\\d*\\.ssm_dt.bias");370 static const std::regex pattern_ssm_a ("blk\\.\\d*\\.ssm_a");371 static const std::regex pattern_ssm_alpha ("blk\\.\\d*\\.ssm_alpha.weight");372 static const std::regex pattern_ssm_beta ("blk\\.\\d*\\.ssm_beta.weight");373 static const std::regex pattern_ssm_beta_alpha ("blk\\.\\d*\\.ssm_ba.weight");374 static const std::regex pattern_r_cache ("cache_r_l\\d*");375 static const std::regex pattern_s_cache ("cache_s_l\\d*");376 static const std::regex pattern_ssm_conv1d ("blk\\.\\d*\\.ssm_conv1d.weight");377 static const std::regex pattern_ssm_out_weight ("blk\\.\\d*\\.ssm_out.weight");378 379 static const std::regex pattern_ffn_up_weight ("blk\\.\\d*\\.ffn_up(_exps)?.weight");380 static const std::regex pattern_ffn_up_bias ("blk\\.\\d*\\.ffn_up(_exps)?.bias");381 static const std::regex pattern_ffn_gate_weight ("blk\\.\\d*\\.ffn_gate(_exps)?.weight");382 static const std::regex pattern_ffn_gate_bias ("blk\\.\\d*\\.ffn_gate(_exps)?.bias");383 static const std::regex pattern_ffn_gate_up_weight("blk\\.\\d*\\.ffn_gate_up(_exps)?.weight");384 static const std::regex pattern_ffn_down_weight ("blk\\.\\d*\\.ffn_down(_exps)?.weight");385 static const std::regex pattern_ffn_down_bias ("blk\\.\\d*\\.ffn_down.bias");386 static const std::regex pattern_ffn_down_exps_bias("blk\\.\\d*\\.ffn_down_exps.bias");387 388 static const std::regex pattern_output_weight("output\\.weight");389 static const std::regex pattern_output_bias ("output\\.bias");390 391 struct tensor_config {392 ggml_backend_meta_split_axis axis;393 394 const ggml_tensor * tensor_axis_0;395 396 uint32_t il;397 size_t rotation; // when assigning tensor slices, rotate how the rounding is done for more even allocation398 };399 400 auto get_tensor_config_impl = [&](401 const ggml_backend_meta_split_axis axis, const std::string & suffix = "", const std::string & suffix_fallback = "") -> tensor_config {402 // the layers in a tensor can be inhomogeneous, if the pattern is cleanly divided by the number of GPUs there can be aliasing effects,403 // count only the same type of previous layers to avoid this404 auto get_il_eff = [&](const size_t il){405 size_t ret = 0;406 const bool il_is_recr = hparams.is_recr(il);407 const bool il_is_swa = hparams.is_swa(il);408 for (size_t il_prev = 0; il_prev < il; il_prev++) {409 ret += hparams.is_recr(il_prev) == il_is_recr && hparams.is_swa(il_prev) == il_is_swa;410 }411 return ret;412 };413 414 uint32_t il;415 std::string prefix;416 size_t rotation;417 if (tensor_name.substr(0, 4) == "blk.") {418 const size_t length_prefix = tensor_name.find('.', 4);419 GGML_ASSERT(length_prefix != std::string::npos);420 prefix = tensor_name.substr(0, length_prefix + 1);421 il = std::stoull(tensor_name.substr(4, length_prefix));422 rotation = get_il_eff(il) % ud->n_devices;423 } else if (tensor_name.substr(0, 6) == "cache_") {424 const size_t layer_index_start = tensor_name.find("_l", 6);425 GGML_ASSERT(layer_index_start != std::string::npos);426 il = std::stoull(tensor_name.substr(layer_index_start + 2));427 prefix = "blk." + std::to_string(il) + ".";428 rotation = get_il_eff(il) % ud->n_devices;429 } else {430 il = 0;431 rotation = hparams.n_layer() % ud->n_devices;432 }433 const ggml_tensor * tensor_axis_0 = suffix.empty() ? tensor : ud->model->get_tensor((prefix + suffix).c_str());434 if (tensor_axis_0 == nullptr) {435 GGML_ASSERT(!suffix_fallback.empty());436 tensor_axis_0 = ud->model->get_tensor((prefix + suffix_fallback).c_str());437 }438 GGML_ASSERT(tensor_axis_0 != nullptr);439 return {axis, tensor_axis_0, il, rotation};440 };441 442 auto get_tensor_config = [&]() -> tensor_config {443 // standard attention444 if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_kv_weight)) {445 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");446 }447 if (std::regex_match(tensor_name, pattern_q_bias) || std::regex_match(tensor_name, pattern_kv_bias)) {448 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight", "ssm_out.weight");449 }450 if (std::regex_match(tensor_name, pattern_qkv_weight)) {451 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");452 }453 if ( std::regex_match(tensor_name, pattern_qkv_bias)) {454 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight", "ssm_out.weight");455 }456 if (std::regex_match(tensor_name, pattern_qk_norm)) {457 return get_tensor_config_impl(tensor->ne[1] == 1 ? GGML_BACKEND_SPLIT_AXIS_MIRRORED : GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight");458 }459 if (std::regex_match(tensor_name, pattern_kv_cache) || std::regex_match(tensor_name, pattern_attn_sinks)) {460 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "attn_output.weight");461 }462 if (std::regex_match(tensor_name, pattern_attn_out_weight)) {463 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);464 }465 if (std::regex_match(tensor_name, pattern_attn_out_bias)) {466 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);467 }468 469 if (std::regex_match(tensor_name, pattern_attn_gate_weight)) {470 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "attn_output.weight", "ssm_out.weight");471 }472 if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a)) {473 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");474 }475 if (std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta) ||476 std::regex_match(tensor_name, pattern_ssm_beta_alpha)) {477 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");478 }479 if (std::regex_match(tensor_name, pattern_r_cache) || std::regex_match(tensor_name, pattern_s_cache)) {480 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ssm_out.weight");481 }482 if (std::regex_match(tensor_name, pattern_ssm_conv1d)) {483 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ssm_out.weight");484 }485 if (std::regex_match(tensor_name, pattern_ssm_out_weight)) {486 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);487 }488 489 // FFN490 if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_gate_weight)) {491 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight");492 }493 if (std::regex_match(tensor_name, pattern_ffn_up_bias) || std::regex_match(tensor_name, pattern_ffn_gate_bias)) {494 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight");495 }496 if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {497 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1, "ffn_down.weight", "ffn_down_exps.weight");498 }499 if (std::regex_match(tensor_name, pattern_ffn_down_weight)) {500 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0, "ffn_down.weight", "ffn_down_exps.weight");501 }502 if (std::regex_match(tensor_name, pattern_ffn_down_bias)) {503 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);504 }505 if (std::regex_match(tensor_name, pattern_ffn_down_exps_bias)) {506 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_PARTIAL);507 }508 509 // output510 if (std::regex_match(tensor_name, pattern_output_weight)) {511 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_1);512 }513 if (std::regex_match(tensor_name, pattern_output_bias)) {514 const ggml_tensor * output_weight = ud->model->get_tensor("output.weight");515 GGML_ASSERT(output_weight != nullptr);516 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_0);517 }518 519 // everything else520 return get_tensor_config_impl(GGML_BACKEND_SPLIT_AXIS_MIRRORED);521 };522 523 auto get_split_segments = [&](int axis, uint32_t il) -> std::vector<std::pair<int64_t, uint32_t>> {524 if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE) {525 const int64_t head_k_dim = hparams.ssm_d_state;526 const int64_t head_v_dim = hparams.ssm_d_state;527 const int64_t n_k_heads = hparams.ssm_n_group;528 const int64_t n_v_heads = hparams.ssm_dt_rank;529 const int64_t key_dim = head_k_dim * n_k_heads;530 const int64_t value_dim = head_v_dim * n_v_heads;531 532 // both Qwen 3 Next and Qwen 3.5 support n_v_heads > n_k_heads but the broadcasting pattern is different:533 // - Qwen 3 Next: [k0_v0, k0_v1, k1_v2, k1_v3] (this is the default split pattern)534 // - Qwen 3.5: [k0_v0, k1_v1, k0_v2, k1_v3] (needs segmenting of V on the scale of K to get the correct pattern)535 if (ud->model->arch == LLM_ARCH_QWEN3NEXT) {536 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_ssm_conv1d)) {537 GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);538 return {{key_dim, 2}, {value_dim, 1}};539 }540 } else {541 const int64_t head_ratio = n_v_heads / n_k_heads;542 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_ssm_conv1d)) {543 GGML_ASSERT(tensor->ne[axis] == 2*key_dim + value_dim);544 return {{key_dim, 2 + head_ratio}};545 }546 if (std::regex_match(tensor_name, pattern_attn_gate_weight) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {547 return {{key_dim, head_ratio}};548 }549 if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||550 std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) {551 return {{n_k_heads, head_ratio}};552 }553 if (std::regex_match(tensor_name, pattern_r_cache)) {554 return {{key_dim * (hparams.ssm_d_conv - 1), 2 + head_ratio}};555 }556 if (std::regex_match(tensor_name, pattern_s_cache)) {557 return {{n_k_heads * head_v_dim * head_v_dim, head_ratio}};558 }559 }560 561 // the FFN is the same for Qwen 3 Next and Qwen 3.5:562 if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {563 const int64_t n_ff_exp = hparams.n_ff_exp;564 GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);565 return {{n_ff_exp, 2}};566 }567 return {{tensor->ne[axis], 1}};568 }569 570 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {571 const int64_t n_embd = hparams.n_embd;572 const int64_t n_embd_gqa = hparams.n_embd_v_gqa(il);573 GGML_ASSERT(hparams.n_embd_k_gqa() == n_embd_gqa);574 GGML_ASSERT(tensor->ne[axis] == n_embd + 2*n_embd_gqa);575 return {{n_embd, 1}, {n_embd_gqa, 2}};576 }577 if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias)) {578 const int64_t n_ff = hparams.n_ff(il);579 // some models such as Phi 3 have fused up + gate tensors named "up" tensors, which need to be segmented580 if (tensor->ne[axis] == 2*n_ff) {581 return {{n_ff, 2}};582 }583 return {{tensor->ne[axis], 1}};584 }585 if (std::regex_match(tensor_name, pattern_ffn_gate_up_weight)) {586 const int64_t n_ff_exp = hparams.n_ff_exp;587 GGML_ASSERT(tensor->ne[axis] == 2*n_ff_exp);588 return {{n_ff_exp, 2}};589 }590 return {{tensor->ne[axis], 1}};591 };592 593 auto get_split_granularity = [&](int64_t blck_size, uint32_t il, const std::vector<std::pair<int64_t, uint32_t>> & segments) -> std::vector<int64_t> {594 // for better performance it may make sense to round up blck_size to a higher power of 2 so that more efficient kernels can be used595 if (hparams.is_recr(il)) {596 // linear attention597 const int64_t head_dim = hparams.ssm_d_state;598 const int64_t blck_size_perf = std::lcm(blck_size, 128);599 const int64_t granularity_qkv = std::lcm(blck_size_perf, head_dim);600 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_attn_gate_weight) ||601 std::regex_match(tensor_name, pattern_ssm_conv1d) || std::regex_match(tensor_name, pattern_ssm_out_weight)) {602 return std::vector<int64_t>(segments.size(), granularity_qkv);603 }604 if (std::regex_match(tensor_name, pattern_ssm_dt) || std::regex_match(tensor_name, pattern_ssm_a) ||605 std::regex_match(tensor_name, pattern_ssm_alpha) || std::regex_match(tensor_name, pattern_ssm_beta)) {606 return std::vector<int64_t>(segments.size(), granularity_qkv / head_dim);607 }608 if (std::regex_match(tensor_name, pattern_ssm_beta_alpha)) {609 return std::vector<int64_t>(segments.size(), 2 * (granularity_qkv / head_dim));610 }611 if (std::regex_match(tensor_name, pattern_r_cache)) {612 return std::vector<int64_t>(segments.size(), granularity_qkv * (hparams.ssm_d_conv - 1));613 }614 if (std::regex_match(tensor_name, pattern_s_cache)) {615 return std::vector<int64_t>(segments.size(), granularity_qkv * head_dim);616 }617 } else {618 // regular attention619 const uint32_t n_gqa = hparams.n_gqa(il);620 const uint32_t n_embd_q = n_gqa * hparams.n_embd_head_k(il);621 622 // to handle head sizes like 80, only increase granularity while it doesn't cause underutilization623 int64_t blck_size_perf = blck_size;624 while (blck_size_perf < 128 && blck_size_perf*ud->n_devices < n_embd_q) {625 blck_size_perf *= 2;626 }627 628 if (std::regex_match(tensor_name, pattern_attn_sinks)) {629 GGML_ASSERT(segments.size() == 1);630 return {std::lcm(n_embd_q, blck_size_perf)/n_embd_q * n_gqa};631 }632 633 const int64_t granularity_q = std::lcm(n_embd_q, blck_size_perf);634 if (std::regex_match(tensor_name, pattern_q_weight) || std::regex_match(tensor_name, pattern_q_bias)) {635 GGML_ASSERT(segments.size() == 1);636 // some models have Q gate tensors, for those cases the granularity needs to be doubled:637 if (ud->model->arch == LLM_ARCH_QWEN3NEXT || ud->model->arch == LLM_ARCH_QWEN35 || ud->model->arch == LLM_ARCH_QWEN35MOE) {638 return {std::lcm(2*n_embd_q, blck_size_perf)};639 }640 return {granularity_q};641 }642 if (std::regex_match(tensor_name, pattern_attn_out_weight)) {643 GGML_ASSERT(segments.size() == 1);644 return {granularity_q};645 }646 647 const int64_t granularity_kv = granularity_q / n_gqa;648 if (std::regex_match(tensor_name, pattern_kv_weight) ||649 std::regex_match(tensor_name, pattern_kv_bias) ||650 std::regex_match(tensor_name, pattern_kv_cache)) {651 GGML_ASSERT(segments.size() == 1);652 return {granularity_kv};653 }654 if (std::regex_match(tensor_name, pattern_qkv_weight) || std::regex_match(tensor_name, pattern_qkv_bias)) {655 GGML_ASSERT(segments.size() == 2);656 return {granularity_q, granularity_kv};657 }658 }659 660 // FFN661 if (std::regex_match(tensor_name, pattern_ffn_up_weight) || std::regex_match(tensor_name, pattern_ffn_up_bias) ||662 std::regex_match(tensor_name, pattern_ffn_gate_weight) || std::regex_match(tensor_name, pattern_ffn_gate_bias) ||663 std::regex_match(tensor_name, pattern_ffn_gate_up_weight) || std::regex_match(tensor_name, pattern_ffn_down_weight)) {664 const int64_t blck_size_perf = std::lcm(blck_size, 128);665 GGML_ASSERT(segments.size() == 1);666 return {blck_size_perf};667 }668 669 // everything else670 GGML_ASSERT(segments.size() == 1);671 return {1};672 };673 674 ggml_backend_meta_split_state split_state;675 memset(&split_state, 0, sizeof(split_state));676 tensor_config tc = get_tensor_config();677 split_state.axis = tc.axis;678 if (split_state.axis >= 0 && split_state.axis < GGML_MAX_DIMS) {679 const int64_t blck_size = ggml_blck_size(tc.tensor_axis_0->type);680 const float * tensor_split = ud->model->tensor_split();681 std::vector<float> tensor_split_scan;682 tensor_split_scan.reserve(ud->n_devices);683 for (size_t j = 0; j < ud->n_devices; j++) {684 tensor_split_scan.push_back(tensor_split == nullptr ? 0.0f : tensor_split[(j + tc.rotation) % ud->n_devices]);685 if (j > 0) {686 tensor_split_scan[j] += tensor_split_scan[j - 1];687 }688 }689 const std::vector<std::pair<int64_t, uint32_t>> segments = get_split_segments(split_state.axis, tc.il);690 const std::vector<int64_t> granularity = get_split_granularity(blck_size, tc.il, segments);691 for (size_t is = 0; is < segments.size(); is++) {692 const int64_t ne_s = segments[is].first;693 const uint32_t nr_s = segments[is].second;694 const int64_t g_s = granularity[is];695 int64_t low = 0;696 size_t j = 0;697 for (; j < ud->n_devices - 1; j++) {698 int64_t high = tensor_split_scan.back() == 0.0f ?699 ne_s * (j+1)/ud->n_devices : ne_s * tensor_split_scan[j]/tensor_split_scan.back();700 if (high % g_s != 0) {701 high -= high % g_s;702 }703 split_state.ne[is*ud->n_devices + (j + tc.rotation) % ud->n_devices] = high - low;704 low = high;705 }706 split_state.ne[is*ud->n_devices + (j + tc.rotation) % ud->n_devices] = ne_s - low;707 split_state.nr[is] = nr_s;708 }709 split_state.n_segments = segments.size();710 } else {711 memset(split_state.ne, 0, sizeof(split_state.ne));712 split_state.nr[0] = 1;713 split_state.n_segments = 1;714 }715 return split_state;716 GGML_UNUSED(userdata);717}718 719const char * llm_type_name(llm_type type) {720 switch (type) {721 case LLM_TYPE_14M: return "14M";722 case LLM_TYPE_17M: return "17M";723 case LLM_TYPE_22M: return "22M";724 case LLM_TYPE_33M: return "33M";725 case LLM_TYPE_47M: return "47M";726 case LLM_TYPE_60M: return "60M";727 case LLM_TYPE_70M: return "70M";728 case LLM_TYPE_80M: return "80M";729 case LLM_TYPE_109M: return "109M";730 case LLM_TYPE_137M: return "137M";731 case LLM_TYPE_140M: return "140M";732 case LLM_TYPE_149M: return "149M";733 case LLM_TYPE_160M: return "160M";734 case LLM_TYPE_190M: return "190M";735 case LLM_TYPE_220M: return "220M";736 case LLM_TYPE_230M: return "230M";737 case LLM_TYPE_250M: return "250M";738 case LLM_TYPE_256M: return "256M";739 case LLM_TYPE_270M: return "270M";740 case LLM_TYPE_335M: return "335M";741 case LLM_TYPE_350M: return "350M";742 case LLM_TYPE_360M: return "360M";743 case LLM_TYPE_395M: return "395M";744 case LLM_TYPE_410M: return "410M";745 case LLM_TYPE_450M: return "450M";746 case LLM_TYPE_475M: return "475M";747 case LLM_TYPE_558M: return "558M";748 case LLM_TYPE_700M: return "700M";749 case LLM_TYPE_770M: return "770M";750 case LLM_TYPE_780M: return "780M";751 case LLM_TYPE_950M: return "950M";752 case LLM_TYPE_0_3B: return "0.3B";753 case LLM_TYPE_0_5B: return "0.5B";754 case LLM_TYPE_0_6B: return "0.6B";755 case LLM_TYPE_0_8B: return "0.8B";756 case LLM_TYPE_1B: return "1B";757 case LLM_TYPE_1_2B: return "1.2B";758 case LLM_TYPE_1_3B: return "1.3B";759 case LLM_TYPE_1_4B: return "1.4B";760 case LLM_TYPE_1_5B: return "1.5B";761 case LLM_TYPE_1_6B: return "1.6B";762 case LLM_TYPE_1_7B: return "1.7B";763 case LLM_TYPE_1_8B: return "1.8B";764 case LLM_TYPE_2B: return "2B";765 case LLM_TYPE_2_6B: return "2.6B";766 case LLM_TYPE_2_8B: return "2.8B";767 case LLM_TYPE_2_9B: return "2.9B";768 case LLM_TYPE_3B: return "3B";769 case LLM_TYPE_4B: return "4B";770 case LLM_TYPE_6B: return "6B";771 case LLM_TYPE_6_9B: return "6.9B";772 case LLM_TYPE_7B: return "7B";773 case LLM_TYPE_8B: return "8B";774 case LLM_TYPE_9B: return "9B";775 case LLM_TYPE_11B: return "11B";776 case LLM_TYPE_12B: return "12B";777 case LLM_TYPE_13B: return "13B";778 case LLM_TYPE_14B: return "14B";779 case LLM_TYPE_15B: return "15B";780 case LLM_TYPE_16B: return "16B";781 case LLM_TYPE_20B: return "20B";782 case LLM_TYPE_26B: return "26B";783 case LLM_TYPE_27B: return "27B";784 case LLM_TYPE_30B: return "30B";785 case LLM_TYPE_31B: return "31B";786 case LLM_TYPE_32B: return "32B";787 case LLM_TYPE_34B: return "34B";788 case LLM_TYPE_35B: return "35B";789 case LLM_TYPE_36B: return "36B";790 case LLM_TYPE_40B: return "40B";791 case LLM_TYPE_65B: return "65B";792 case LLM_TYPE_70B: return "70B";793 case LLM_TYPE_120B: return "120B";794 case LLM_TYPE_142B: return "142B";795 case LLM_TYPE_236B: return "236B";796 case LLM_TYPE_290B: return "290B";797 case LLM_TYPE_314B: return "314B";798 case LLM_TYPE_405B: return "405B";799 case LLM_TYPE_671B: return "671B";800 case LLM_TYPE_SMALL: return "0.1B";801 case LLM_TYPE_MEDIUM: return "0.4B";802 case LLM_TYPE_LARGE: return "0.8B";803 case LLM_TYPE_XL: return "1.5B";804 case LLM_TYPE_A1_7B: return "A1.7B";805 case LLM_TYPE_A2_7B: return "A2.7B";806 case LLM_TYPE_8x7B: return "8x7B";807 case LLM_TYPE_8x22B: return "8x22B";808 case LLM_TYPE_16x12B: return "16x12B";809 case LLM_TYPE_16x3_8B: return "16x3.8B";810 case LLM_TYPE_10B_128x3_66B: return "10B+128x3.66B";811 case LLM_TYPE_57B_A14B: return "57B.A14B";812 case LLM_TYPE_17B_16E: return "17Bx16E (Scout)";813 case LLM_TYPE_17B_128E: return "17Bx128E (Maverick)";814 case LLM_TYPE_A13B: return "A13B";815 case LLM_TYPE_7B_A1B: return "7B.A1B";816 case LLM_TYPE_8B_A1B: return "8B.A1B";817 case LLM_TYPE_12B_A2_5B: return "12B.A2.5B";818 case LLM_TYPE_16B_A1B: return "16B.A1B";819 case LLM_TYPE_21B_A3B: return "21B.A3B";820 case LLM_TYPE_24B_A2B: return "24B.A2B";821 case LLM_TYPE_26B_A4B: return "26B.A4B";822 case LLM_TYPE_30B_A3B: return "30B.A3B";823 case LLM_TYPE_31B_A3_5B: return "31B.A3.5B";824 case LLM_TYPE_35B_A3B: return "35B.A3B";825 case LLM_TYPE_48B_A3B: return "48B.A3B";826 case LLM_TYPE_80B_A3B: return "80B.A3B";827 case LLM_TYPE_100B_A6B: return "100B.A6B";828 case LLM_TYPE_102B_A12B: return "102B.A12B";829 case LLM_TYPE_106B_A12B: return "106B.A12B";830 case LLM_TYPE_118B_A8B: return "118B.A8B";831 case LLM_TYPE_120B_A12B: return "120B.A12B";832 case LLM_TYPE_122B_A10B: return "122B.A10B";833 case LLM_TYPE_196B_A11B: return "196B.A11B";834 case LLM_TYPE_230B_A10B: return "230B.A10B";835 case LLM_TYPE_428B_A23B: return "428B.A23B";836 case LLM_TYPE_235B_A22B: return "235B.A22B";837 case LLM_TYPE_300B_A47B: return "300B.A47B";838 case LLM_TYPE_310B_A15B: return "310B.A15B";839 case LLM_TYPE_355B_A32B: return "355B.A32B";840 case LLM_TYPE_397B_A17B: return "397B.A17B";841 case LLM_TYPE_685B_A37B: return "685B.A37B";842 case LLM_TYPE_744B_A40B: return "744B.A40B";843 case LLM_TYPE_E2B: return "E2B";844 case LLM_TYPE_E4B: return "E4B";845 default: return "?B";846 }847}848 849static const char * llama_expert_gating_func_name(llama_expert_gating_func_type type) {850 switch (type) {851 case LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX: return "softmax";852 case LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID: return "sigmoid";853 case LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS: return "sqrtsoftplus";854 default: return "unknown";855 }856}857 858static const std::map<llama_rope_scaling_type, const char *> LLAMA_ROPE_SCALING_TYPES = {859 { LLAMA_ROPE_SCALING_TYPE_NONE, "none" },860 { LLAMA_ROPE_SCALING_TYPE_LINEAR, "linear" },861 { LLAMA_ROPE_SCALING_TYPE_YARN, "yarn" },862 { LLAMA_ROPE_SCALING_TYPE_LONGROPE, "longrope" },863};864 865std::string llama_rope_scaling_type_name(llama_rope_scaling_type rope_scaling_type) {866 return LLAMA_ROPE_SCALING_TYPES.at(rope_scaling_type);867}868 869static llama_rope_scaling_type llama_rope_scaling_type_from_string(const std::string & name) {870 for (const auto & kv : LLAMA_ROPE_SCALING_TYPES) {871 if (kv.second == name) {872 return (llama_rope_scaling_type) kv.first;873 }874 }875 876 return LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;877}878 879// Maps the GGUF `<arch>.hidden_activation` string to the FFN op type used by the880// graph builders. Only gated activations that map cleanly to llm_ffn_op_type are881// listed; unrecognized values fall back to GeGLU, which matches the historical882// default for ModernBert-style architectures.883static const std::map<std::string, llm_ffn_op_type> LLM_FFN_OP_TYPES_FROM_STRING = {884 { "gelu", LLM_FFN_GEGLU },885 { "geglu", LLM_FFN_GEGLU },886 { "silu", LLM_FFN_SWIGLU },887 { "swish", LLM_FFN_SWIGLU },888 { "swiglu", LLM_FFN_SWIGLU },889 { "relu", LLM_FFN_RELU },890 { "reglu", LLM_FFN_REGLU },891};892 893llm_ffn_op_type llm_ffn_op_type_from_string(const std::string & name, llm_ffn_op_type fallback) {894 const auto it = LLM_FFN_OP_TYPES_FROM_STRING.find(name);895 if (it != LLM_FFN_OP_TYPES_FROM_STRING.end()) {896 return it->second;897 }898 return fallback;899}900 901// CPU: ACCEL -> GPU host -> CPU extra -> CPU902static buft_list_t make_cpu_buft_list(const std::vector<llama_device> & devices, bool use_extra_bufts, bool no_host) {903 buft_list_t buft_list;904 905 // add ACCEL buffer types906 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {907 ggml_backend_dev_t dev = ggml_backend_dev_get(i);908 if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_ACCEL) {909 auto * buft = ggml_backend_dev_buffer_type(dev);910 // skip911 if (buft != ggml_backend_cpu_buffer_type()) {912 buft_list.emplace_back(dev, buft);913 }914 }915 }916 917 // add a host buffer type918 // storing the tensors in a host buffer is useful when the processing of large batches919 // is offloaded to a GPU device, since it reduces the time spent on data transfers920 // generally, this will be done using the first device in the list921 // a better approach would be to handle this on a weight-by-weight basis using the offload_op922 // function of the device to determine if it would benefit from being stored in a host buffer923 if (!no_host) {924 for (const auto & dev : devices) {925 ggml_backend_buffer_type_t buft = ggml_backend_dev_host_buffer_type(dev.dev);926 if (buft) {927 buft_list.emplace_back(dev.dev, buft);928 break;929 }930 }931 }932 933 // add extra buffer types934 if (use_extra_bufts) {935 auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU);936 if (cpu_dev == nullptr) {937 throw std::runtime_error(format("%s: no CPU backend found", __func__));938 }939 940 auto * cpu_reg = ggml_backend_dev_backend_reg(cpu_dev);941 auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)942 ggml_backend_reg_get_proc_address(cpu_reg, "ggml_backend_dev_get_extra_bufts");943 if (ggml_backend_dev_get_extra_bufts_fn) {944 ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(cpu_dev);945 while (extra_bufts && *extra_bufts) {946 buft_list.emplace_back(cpu_dev, *extra_bufts);947 ++extra_bufts;948 }949 }950 }951 952 // add the CPU buffer type953 for (size_t i = 0; i < ggml_backend_dev_count(); ++i) {954 ggml_backend_dev_t dev = ggml_backend_dev_get(i);955 if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {956 buft_list.emplace_back(dev, ggml_backend_dev_buffer_type(dev));957 }958 }959 960 return buft_list;961}962 963// GPU: split if LLAMA_SPLIT_MODE_ROW -> GPU964static buft_list_t make_gpu_buft_list(ggml_backend_dev_t dev, llama_split_mode split_mode, const float * tensor_split) {965 buft_list_t buft_list;966 967 // add the device split buffer type if requested and available968 if (split_mode == LLAMA_SPLIT_MODE_ROW) {969 ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);970 auto ggml_backend_split_buffer_type_fn = (ggml_backend_split_buffer_type_t)971 ggml_backend_reg_get_proc_address(reg, "ggml_backend_split_buffer_type");972 if (ggml_backend_split_buffer_type_fn) {973 size_t dev_index = [&]() {974 auto * reg = ggml_backend_dev_backend_reg(dev);975 for (size_t i = 0; i < ggml_backend_reg_dev_count(reg); ++i) {976 if (ggml_backend_reg_dev_get(reg, i) == dev) {977 return i;978 }979 }980 throw std::runtime_error(format("device %s not found in its backend reg", ggml_backend_dev_name(dev)));981 }();982 auto * buft = ggml_backend_split_buffer_type_fn(dev_index, tensor_split);983 if (buft != nullptr) {984 buft_list.emplace_back(dev, buft);985 }986 } else {987 throw std::runtime_error(format("device %s does not support split buffers", ggml_backend_dev_name(dev)));988 }989 }990 991 // add the device default buffer type992 buft_list.emplace_back(dev, ggml_backend_dev_buffer_type(dev));993 994 // add the device extra buffer type (if any)995 ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);996 if (reg) {997 auto ggml_backend_dev_get_extra_bufts_fn = (ggml_backend_dev_get_extra_bufts_t)998 ggml_backend_reg_get_proc_address(reg, "ggml_backend_dev_get_extra_bufts");999 1000 if (ggml_backend_dev_get_extra_bufts_fn) {1001 ggml_backend_buffer_type_t * extra_bufts = ggml_backend_dev_get_extra_bufts_fn(dev);1002 while (extra_bufts && *extra_bufts) {1003 buft_list.emplace_back(dev, *extra_bufts);1004 ++extra_bufts;1005 }1006 }1007 }1008 1009 return buft_list;1010}1011 1012struct llama_model::impl {1013 impl() = default;1014 ~impl() = default;1015 1016 uint64_t n_elements = 0;1017 1018 size_t n_bytes = 0;1019 1020 std::string desc_str;1021 1022 llama_ftype ftype = LLAMA_FTYPE_ALL_F32;1023 1024 // model memory mapped files1025 llama_mmaps mappings;1026 1027 // objects representing data potentially being locked in memory1028 llama_mlocks mlock_bufs;1029 llama_mlocks mlock_mmaps;1030 1031 // contexts where the model tensors metadata is stored as well as the corresponding buffers:1032 std::vector<std::pair<ggml_context_ptr, std::vector<ggml_backend_buffer_ptr>>> ctxs_bufs;1033 1034 buft_list_t cpu_buft_list;1035 std::map<ggml_backend_dev_t, buft_list_t> gpu_buft_list;1036 1037 struct layer_dev {1038 ggml_backend_dev_t dev;1039 buft_list_t * buft_list;1040 };1041 1042 layer_dev dev_input = {};1043 layer_dev dev_output = {};1044 std::vector<layer_dev> dev_layer;1045 1046 bool has_tensor_overrides;1047 1048 std::vector<float> tensor_split_owned;1049};1050 1051llama_model::llama_model(const llama_model_params & params) : params(params), pimpl(std::make_unique<impl>()) {1052 if (params.tensor_split != nullptr) {1053 // llama_model_params stores tensor_split as a borrowed pointer, but the model1054 // may need it later for tensor-parallel KV-cache split metadata.1055 pimpl->tensor_split_owned.assign(params.tensor_split, params.tensor_split + llama_max_devices());1056 this->params.tensor_split = pimpl->tensor_split_owned.data();1057 }1058 pimpl->has_tensor_overrides = params.tensor_buft_overrides && params.tensor_buft_overrides[0].pattern;1059}1060 1061llama_model::~llama_model() {1062 for (auto * lora : loras) {1063 delete lora;1064 }1065}1066 1067void llama_model_base::load_stats(llama_model_loader & ml) {1068 pimpl->n_elements = ml.n_elements;1069 pimpl->n_bytes = ml.n_bytes;1070}1071 1072void llama_model_base::load_hparams(llama_model_loader & ml) {1073 const gguf_context * ctx = ml.metadata;1074 1075 // get metadata as string1076 for (int i = 0; i < gguf_get_n_kv(ctx); i++) {1077 gguf_type type = gguf_get_kv_type(ctx, i);1078 if (type == GGUF_TYPE_ARRAY) {1079 continue;1080 }1081 const char * name = gguf_get_key(ctx, i);1082 const std::string value = gguf_kv_to_str(ctx, i);1083 gguf_kv.emplace(name, value);1084 }1085 1086 // get general kv1087 ml.get_key(LLM_KV_GENERAL_NAME, name, false);1088 1089 // everything past this point is not vocab-related1090 // for CLIP models, we only need to load tensors, no hparams1091 if (hparams.vocab_only || ml.get_arch() == LLM_ARCH_CLIP) {1092 return;1093 }1094 1095 ml.get_key(LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train);1096 ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd);1097 ml.get_key(LLM_KV_EMBEDDING_LENGTH_OUT, hparams.n_embd_out_impl, false);1098 ml.get_key(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn, false);1099 ml.get_key(LLM_KV_POOLING_TYPE, hparams.pooling_type, false);1100 ml.get_key(LLM_KV_BLOCK_COUNT, hparams.n_layer_all);1101 GGML_ASSERT(hparams.n_layer_all > 0 && hparams.n_layer_all <= LLAMA_MAX_LAYERS);1102 ml.get_key(LLM_KV_EXPERT_COUNT, hparams.n_expert, false);1103 ml.get_key(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used, false);1104 ml.get_key(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups, false);1105 ml.get_key(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used, false);1106 1107 if (arch == LLM_ARCH_HUNYUAN_VL || arch == LLM_ARCH_HUNYUAN_DENSE) {1108 if (hparams.n_expert <= 1) {1109 hparams.n_expert = 0;1110 hparams.n_expert_used = 0;1111 }1112 }1113 1114 if (arch == LLM_ARCH_WAVTOKENIZER_DEC) {1115 ml.get_key(LLM_KV_FEATURES_LENGTH, hparams.n_embd);1116 ml.get_key(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd_out_impl);1117 1118 ml.get_key(LLM_KV_POSNET_EMBEDDING_LENGTH, hparams.posnet.n_embd);1119 ml.get_key(LLM_KV_POSNET_BLOCK_COUNT, hparams.posnet.n_layer);1120 1121 ml.get_key(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, hparams.convnext.n_embd);1122 ml.get_key(LLM_KV_CONVNEXT_BLOCK_COUNT, hparams.convnext.n_layer);1123 }1124 1125 GGML_ASSERT(hparams.n_expert <= LLAMA_MAX_EXPERTS);1126 GGML_ASSERT(hparams.n_expert_used <= hparams.n_expert);1127 if (hparams.n_expert > 0) {1128 GGML_ASSERT(hparams.n_expert_used > 0);1129 GGML_ASSERT(hparams.n_expert_groups < hparams.n_expert);1130 if (hparams.n_expert_groups > 1) {1131 GGML_ASSERT(hparams.n_expert % hparams.n_expert_groups == 0);1132 GGML_ASSERT(hparams.n_group_used > 0);1133 GGML_ASSERT(hparams.n_group_used < hparams.n_expert_groups);1134 }1135 } else {1136 GGML_ASSERT(hparams.n_expert_used == 0);1137 GGML_ASSERT(hparams.n_expert_groups == 0);1138 }1139 1140 std::fill(hparams.n_head_arr.begin(), hparams.n_head_arr.end(), 0);1141 std::fill(hparams.n_head_kv_arr.begin(), hparams.n_head_kv_arr.end(), 0);1142 std::fill(hparams.n_ff_arr.begin(), hparams.n_ff_arr.end(), 0);1143 1144 std::fill(hparams.rope_sections.begin(), hparams.rope_sections.end(), 0);1145 std::fill(hparams.is_swa_impl.begin(), hparams.is_swa_impl.end(), 0);1146 std::fill(hparams.is_recr_impl.begin(), hparams.is_recr_impl.end(), llm_arch_is_recurrent(ml.get_arch()) ? 1 : 0);1147 std::fill(hparams.is_indexer_full_impl.begin(), hparams.is_indexer_full_impl.end(), 0);1148 1149 std::fill(hparams.xielu_alpha_n.begin(), hparams.xielu_alpha_n.end(), 0.0f);1150 std::fill(hparams.xielu_alpha_p.begin(), hparams.xielu_alpha_p.end(), 0.0f);1151 std::fill(hparams.xielu_beta.begin(), hparams.xielu_beta.end(), 0.0f);1152 std::fill(hparams.xielu_eps.begin(), hparams.xielu_eps.end(), 0.0f);1153 1154 std::fill(hparams.swiglu_clamp_exp.begin(), hparams.swiglu_clamp_exp.end(), 0.0f);1155 std::fill(hparams.swiglu_clamp_shexp.begin(), hparams.swiglu_clamp_shexp.end(), 0.0f);1156 1157 ml.get_key_or_arr(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, hparams.n_layer(), false);1158 ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, hparams.n_layer(), false);1159 1160 // Populate deepstack_mapping_arr - initialized to -1 (no deepstack)1161 std::fill(hparams.deepstack_mapping_arr.begin(), hparams.deepstack_mapping_arr.end(), -1);1162 1163 // n_head_kv is optional, default to n_head1164 hparams.n_head_kv_arr = hparams.n_head_arr;1165 1166 ml.get_key_or_arr(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, hparams.n_layer(), false);1167 1168 bool rope_finetuned = false;1169 ml.get_key(LLM_KV_ROPE_SCALING_FINETUNED, rope_finetuned, false);1170 hparams.rope_finetuned = rope_finetuned;1171 1172 hparams.n_ctx_orig_yarn = hparams.n_ctx_train;1173 ml.get_key(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, hparams.n_ctx_orig_yarn, false);1174 1175 // rope_freq_base (optional)1176 hparams.rope_freq_base_train = 10000.0f;1177 ml.get_key(LLM_KV_ROPE_FREQ_BASE, hparams.rope_freq_base_train, false);1178 1179 std::string rope_scaling("linear");1180 ml.get_key(LLM_KV_ROPE_SCALING_TYPE, rope_scaling, false);1181 hparams.rope_scaling_type_train = llama_rope_scaling_type_from_string(rope_scaling);1182 GGML_ASSERT(hparams.rope_scaling_type_train != LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED);1183 1184 // TODO: Handle SWA metadata similarly when models start implementing it1185 // rope_freq_scale (inverse of the kv) is optional1186 float ropescale = 0.0f;1187 if (!ml.get_key(LLM_KV_ROPE_SCALING_FACTOR, ropescale, false)) {1188 // try the old key name1189 ml.get_key(LLM_KV_ROPE_SCALE_LINEAR, ropescale, false);1190 }1191 hparams.rope_freq_scale_train = ropescale == 0.0f ? 1.0f : 1.0f/ropescale;1192 1193 ml.get_key(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor, false);1194 ml.get_key(LLM_KV_ROPE_SCALING_ALPHA, hparams.rope_scaling_alpha, false);1195 1196 // non-transformer models do not have attention heads1197 if (hparams.n_head() > 0) {1198 // gpt-neox n_rot = rotary_pct * (n_embd / n_head)1199 // gpt-j n_rot = rotary_dim1200