echodict/llama.cpp

1#pragma once2 3#include "llama.h"4#include "llama-arch.h"5#include "llama-graph.h"6#include "llama-hparams.h"7#include "llama-memory.h"8#include "llama-vocab.h"9 10#include <map>11#include <memory>12#include <string>13#include <unordered_map>14#include <unordered_set>15#include <vector>16 17struct llama_cparams;18struct llama_ubatch;19struct llama_model_loader;20 21// available models22enum llm_type {23    LLM_TYPE_UNKNOWN,24    LLM_TYPE_14M,25    LLM_TYPE_17M,26    LLM_TYPE_22M,27    LLM_TYPE_33M,28    LLM_TYPE_47M,29    LLM_TYPE_60M,30    LLM_TYPE_70M,31    LLM_TYPE_80M,32    LLM_TYPE_109M,33    LLM_TYPE_137M,34    LLM_TYPE_140M,35    LLM_TYPE_149M,36    LLM_TYPE_160M,37    LLM_TYPE_190M,38    LLM_TYPE_220M,39    LLM_TYPE_250M,40    LLM_TYPE_256M,41    LLM_TYPE_270M,42    LLM_TYPE_335M,43    LLM_TYPE_350M,44    LLM_TYPE_360M,45    LLM_TYPE_395M,46    LLM_TYPE_410M,47    LLM_TYPE_450M,48    LLM_TYPE_475M,49    LLM_TYPE_558M,50    LLM_TYPE_700M,51    LLM_TYPE_770M,52    LLM_TYPE_780M,53    LLM_TYPE_950M,54    LLM_TYPE_0_3B,55    LLM_TYPE_0_5B,56    LLM_TYPE_0_6B,57    LLM_TYPE_0_8B,58    LLM_TYPE_1B,59    LLM_TYPE_1_2B,60    LLM_TYPE_1_3B,61    LLM_TYPE_1_4B,62    LLM_TYPE_1_5B,63    LLM_TYPE_1_6B,64    LLM_TYPE_1_7B,65    LLM_TYPE_1_8B,66    LLM_TYPE_2B,67    LLM_TYPE_2_6B,68    LLM_TYPE_2_8B,69    LLM_TYPE_2_9B,70    LLM_TYPE_3B,71    LLM_TYPE_4B,72    LLM_TYPE_6B,73    LLM_TYPE_6_9B,74    LLM_TYPE_7B,75    LLM_TYPE_8B,76    LLM_TYPE_9B,77    LLM_TYPE_11B,78    LLM_TYPE_12B,79    LLM_TYPE_13B,80    LLM_TYPE_14B,81    LLM_TYPE_15B,82    LLM_TYPE_16B,83    LLM_TYPE_20B,84    LLM_TYPE_26B,85    LLM_TYPE_27B,86    LLM_TYPE_30B,87    LLM_TYPE_31B,88    LLM_TYPE_32B,89    LLM_TYPE_34B,90    LLM_TYPE_35B,91    LLM_TYPE_36B,92    LLM_TYPE_40B,93    LLM_TYPE_65B,94    LLM_TYPE_70B,95    LLM_TYPE_120B,96    LLM_TYPE_142B,97    LLM_TYPE_236B,98    LLM_TYPE_290B,99    LLM_TYPE_314B,100    LLM_TYPE_405B,101    LLM_TYPE_671B,102    LLM_TYPE_SMALL,103    LLM_TYPE_MEDIUM,104    LLM_TYPE_LARGE,105    LLM_TYPE_XL,106    LLM_TYPE_A1_7B,107    LLM_TYPE_A2_7B,108    LLM_TYPE_8x7B,109    LLM_TYPE_8x22B,110    LLM_TYPE_16x12B,111    LLM_TYPE_16x3_8B,112    LLM_TYPE_10B_128x3_66B,113    LLM_TYPE_57B_A14B,114    LLM_TYPE_17B_16E, // llama4 Scout115    LLM_TYPE_17B_128E, // llama4 Maverick116    LLM_TYPE_A13B,117    LLM_TYPE_7B_A1B,118    LLM_TYPE_8B_A1B, // lfm2moe119    LLM_TYPE_16B_A1B,120    LLM_TYPE_21B_A3B, // Ernie MoE small121    LLM_TYPE_24B_A2B, // lfm2moe122    LLM_TYPE_26B_A4B, // Gemma4123    LLM_TYPE_30B_A3B,124    LLM_TYPE_31B_A3_5B,125    LLM_TYPE_35B_A3B, // Qwen3.5126    LLM_TYPE_48B_A3B, // Kimi Linear127    LLM_TYPE_80B_A3B, // Qwen3 Next128    LLM_TYPE_100B_A6B,129    LLM_TYPE_102B_A12B, // Solar-Open130    LLM_TYPE_106B_A12B, // GLM-4.5-Air131    LLM_TYPE_120B_A12B, // Nemotron 3 Super132    LLM_TYPE_122B_A10B, // Qwen3.5133    LLM_TYPE_196B_A11B, // Step3.5-Flash134    LLM_TYPE_230B_A10B, // Minimax M2135    LLM_TYPE_235B_A22B,136    LLM_TYPE_300B_A47B, // Ernie MoE big137    LLM_TYPE_310B_A15B, // /MiMo-V2-Flash138    LLM_TYPE_355B_A32B, // GLM-4.5139    LLM_TYPE_397B_A17B, // Qwen3.5140    LLM_TYPE_744B_A40B, // GLM-5141    LLM_TYPE_E2B,142    LLM_TYPE_E4B,143};144 145std::string llama_rope_scaling_type_name(llama_rope_scaling_type rope_scaling_type);146 147struct llama_layer_posnet {148    // resnet149    struct ggml_tensor * norm1   = nullptr;150    struct ggml_tensor * norm1_b = nullptr;151 152    struct ggml_tensor * conv1   = nullptr;153    struct ggml_tensor * conv1_b = nullptr;154 155    struct ggml_tensor * norm2   = nullptr;156    struct ggml_tensor * norm2_b = nullptr;157 158    struct ggml_tensor * conv2   = nullptr;159    struct ggml_tensor * conv2_b = nullptr;160 161    // attention162    struct ggml_tensor * attn_norm   = nullptr;163    struct ggml_tensor * attn_norm_b = nullptr;164 165    struct ggml_tensor * attn_q   = nullptr;166    struct ggml_tensor * attn_q_b = nullptr;167 168    struct ggml_tensor * attn_k   = nullptr;169    struct ggml_tensor * attn_k_b = nullptr;170 171    struct ggml_tensor * attn_v   = nullptr;172    struct ggml_tensor * attn_v_b = nullptr;173 174    struct ggml_tensor * attn_o   = nullptr;175    struct ggml_tensor * attn_o_b = nullptr;176 177    // normalize178    struct ggml_tensor * norm   = nullptr;179    struct ggml_tensor * norm_b = nullptr;180};181 182struct llama_layer_convnext {183    struct ggml_tensor * dw   = nullptr;184    struct ggml_tensor * dw_b = nullptr;185 186    struct ggml_tensor * norm   = nullptr;187    struct ggml_tensor * norm_b = nullptr;188 189    struct ggml_tensor * pw1   = nullptr;190    struct ggml_tensor * pw1_b = nullptr;191 192    struct ggml_tensor * pw2   = nullptr;193    struct ggml_tensor * pw2_b = nullptr;194 195    struct ggml_tensor * gamma = nullptr;196};197 198struct llama_layer_shortconv {199    struct ggml_tensor * in_proj  = nullptr;200    struct ggml_tensor * conv     = nullptr;201    struct ggml_tensor * out_proj = nullptr;202};203 204struct llama_layer_nextn {205    struct ggml_tensor * eh_proj          = nullptr;206    struct ggml_tensor * embed_tokens     = nullptr;207    struct ggml_tensor * enorm            = nullptr;208    struct ggml_tensor * hnorm            = nullptr;209    struct ggml_tensor * shared_head_head = nullptr;210    struct ggml_tensor * shared_head_norm = nullptr;211};212 213struct llama_layer {214    // normalization215    struct ggml_tensor * attn_norm       = nullptr;216    struct ggml_tensor * attn_norm_b     = nullptr;217    struct ggml_tensor * attn_norm_2     = nullptr;218    struct ggml_tensor * attn_norm_2_b   = nullptr;219    struct ggml_tensor * attn_q_norm     = nullptr;220    struct ggml_tensor * attn_q_norm_b   = nullptr;221    struct ggml_tensor * attn_k_norm     = nullptr;222    struct ggml_tensor * attn_k_norm_b   = nullptr;223    struct ggml_tensor * attn_out_norm   = nullptr;224    struct ggml_tensor * attn_out_norm_b = nullptr;225    struct ggml_tensor * attn_q_a_norm   = nullptr;226    struct ggml_tensor * attn_kv_a_norm  = nullptr;227    struct ggml_tensor * attn_sub_norm   = nullptr;228    struct ggml_tensor * attn_post_norm  = nullptr;229    struct ggml_tensor * ffn_sub_norm    = nullptr;230    struct ggml_tensor * attn_norm_cross = nullptr;231    struct ggml_tensor * attn_norm_enc   = nullptr;232    struct ggml_tensor * ssm_norm        = nullptr;233    struct ggml_tensor * ssm_dt_norm     = nullptr;234    struct ggml_tensor * ssm_b_norm      = nullptr;235    struct ggml_tensor * ssm_c_norm      = nullptr;236 237    // attention238    struct ggml_tensor * wq        = nullptr;239    struct ggml_tensor * wk        = nullptr;240    struct ggml_tensor * wv        = nullptr;241    struct ggml_tensor * wo        = nullptr;242    struct ggml_tensor * wqkv      = nullptr;243    struct ggml_tensor * wq_a      = nullptr;244    struct ggml_tensor * wq_b      = nullptr;245    struct ggml_tensor * wkv_a_mqa = nullptr;246    struct ggml_tensor * wkv_b     = nullptr;247    struct ggml_tensor * wk_b      = nullptr;248    struct ggml_tensor * wv_b      = nullptr;249    struct ggml_tensor * wqkv_b    = nullptr;250    struct ggml_tensor * wo_b      = nullptr;251    struct ggml_tensor * wq_cross  = nullptr;252    struct ggml_tensor * wk_cross  = nullptr;253    struct ggml_tensor * wv_cross  = nullptr;254    struct ggml_tensor * wo_cross  = nullptr;255    struct ggml_tensor * wq_enc    = nullptr;256    struct ggml_tensor * wk_enc    = nullptr;257    struct ggml_tensor * wv_enc    = nullptr;258    struct ggml_tensor * wo_enc    = nullptr;259    struct ggml_tensor * wqkv_gate = nullptr;260 261    // relative position bias262    struct ggml_tensor * attn_rel_b       = nullptr;263    struct ggml_tensor * attn_rel_b_enc   = nullptr;264    struct ggml_tensor * attn_rel_b_cross = nullptr;265 266    // normalization267    struct ggml_tensor * ffn_norm         = nullptr;268    struct ggml_tensor * ffn_norm_b       = nullptr;269    struct ggml_tensor * ffn_post_norm    = nullptr;270    struct ggml_tensor * ffn_post_norm_1  = nullptr; // gemma4271    struct ggml_tensor * ffn_post_norm_2  = nullptr; // gemma4272    struct ggml_tensor * ffn_pre_norm_2   = nullptr; // gemma4273    struct ggml_tensor * layer_out_norm   = nullptr;274    struct ggml_tensor * layer_out_norm_b = nullptr;275    struct ggml_tensor * ffn_norm_exps    = nullptr;276    struct ggml_tensor * ffn_norm_enc     = nullptr;277 278    // ff279    struct ggml_tensor * ffn_gate     = nullptr; // w1280    struct ggml_tensor * ffn_down     = nullptr; // w2281    struct ggml_tensor * ffn_up       = nullptr; // w3282    struct ggml_tensor * ffn_gate_enc = nullptr;283    struct ggml_tensor * ffn_down_enc = nullptr;284    struct ggml_tensor * ffn_up_enc   = nullptr;285 286    // ff MoE287    struct ggml_tensor * ffn_gate_inp      = nullptr;288    struct ggml_tensor * ffn_gate_inp_s    = nullptr; // gemma4289    struct ggml_tensor * ffn_gate_exps     = nullptr;290    struct ggml_tensor * ffn_down_exps     = nullptr;291    struct ggml_tensor * ffn_up_exps       = nullptr;292    struct ggml_tensor * ffn_gate_up_exps  = nullptr;293    struct ggml_tensor * ffn_gate_inp_b    = nullptr;294    struct ggml_tensor * ffn_gate_exps_b   = nullptr;295    struct ggml_tensor * ffn_down_exps_b   = nullptr;296    struct ggml_tensor * ffn_up_exps_b     = nullptr;297    struct ggml_tensor * ffn_gate_up_exps_b = nullptr;298 299    // ff MoE per-expert scales (NVFP4 per-tensor scale2)300    struct ggml_tensor * ffn_gate_exps_s   = nullptr;301    struct ggml_tensor * ffn_down_exps_s   = nullptr;302    struct ggml_tensor * ffn_up_exps_s     = nullptr;303 304    // ff MoE latent proj305    struct ggml_tensor * ffn_latent_down = nullptr;306    struct ggml_tensor * ffn_latent_up   = nullptr;307 308    // ff shared expert (shexp)309    struct ggml_tensor * ffn_gate_inp_shexp = nullptr;310    struct ggml_tensor * ffn_gate_shexp     = nullptr;311    struct ggml_tensor * ffn_down_shexp     = nullptr;312    struct ggml_tensor * ffn_up_shexp       = nullptr;313 314    // ff adjugate experts (chexps)315    struct ggml_tensor * ffn_gate_chexps     = nullptr;316    struct ggml_tensor * ffn_down_chexps     = nullptr;317    struct ggml_tensor * ffn_up_chexps       = nullptr;318 319    // ff bias320    struct ggml_tensor * ffn_gate_b = nullptr;321    struct ggml_tensor * ffn_down_b = nullptr; // b2322    struct ggml_tensor * ffn_up_b   = nullptr; // b3323    struct ggml_tensor * ffn_act    = nullptr;324    struct ggml_tensor * ffn_exp_probs_b = nullptr;325 326    // mamba proj327    struct ggml_tensor * ssm_in  = nullptr;328    struct ggml_tensor * ssm_x   = nullptr;329    struct ggml_tensor * ssm_dt  = nullptr;330    struct ggml_tensor * ssm_out = nullptr;331 332    // mamba333    struct ggml_tensor * ssm_conv1d = nullptr;334    struct ggml_tensor * ssm_a      = nullptr;335    struct ggml_tensor * ssm_d      = nullptr;336 337    // mamba bias338    struct ggml_tensor * ssm_conv1d_b = nullptr;339    struct ggml_tensor * ssm_dt_b     = nullptr;340 341    // qwen3next342    struct ggml_tensor * ssm_beta_alpha = nullptr;343 344    // qwen3.5345    struct ggml_tensor * ssm_alpha = nullptr;346 347    // rwkv348    struct ggml_tensor * time_mix_w1         = nullptr;349    struct ggml_tensor * time_mix_w2         = nullptr;350    struct ggml_tensor * time_mix_lerp_x     = nullptr;351    struct ggml_tensor * time_mix_lerp_w     = nullptr;352    struct ggml_tensor * time_mix_lerp_k     = nullptr;353    struct ggml_tensor * time_mix_lerp_v     = nullptr;354    struct ggml_tensor * time_mix_lerp_r     = nullptr;355    struct ggml_tensor * time_mix_lerp_g     = nullptr;356    struct ggml_tensor * time_mix_lerp_fused = nullptr;357 358    struct ggml_tensor * time_mix_first        = nullptr;359    struct ggml_tensor * time_mix_decay        = nullptr;360    struct ggml_tensor * time_mix_decay_w1     = nullptr;361    struct ggml_tensor * time_mix_decay_w2     = nullptr;362    struct ggml_tensor * time_mix_key          = nullptr;363    struct ggml_tensor * time_mix_key_b        = nullptr;364    struct ggml_tensor * time_mix_value        = nullptr;365    struct ggml_tensor * time_mix_value_b      = nullptr;366    struct ggml_tensor * time_mix_receptance   = nullptr;367    struct ggml_tensor * time_mix_receptance_b = nullptr;368    struct ggml_tensor * time_mix_gate         = nullptr;369 370    // rwkv7371    struct ggml_tensor * time_mix_w0         = nullptr;372    struct ggml_tensor * time_mix_a0         = nullptr;373    struct ggml_tensor * time_mix_a1         = nullptr;374    struct ggml_tensor * time_mix_a2         = nullptr;375    struct ggml_tensor * time_mix_v0         = nullptr;376    struct ggml_tensor * time_mix_v1         = nullptr;377    struct ggml_tensor * time_mix_v2         = nullptr;378    struct ggml_tensor * time_mix_g1         = nullptr;379    struct ggml_tensor * time_mix_g2         = nullptr;380    struct ggml_tensor * time_mix_k_k        = nullptr;381    struct ggml_tensor * time_mix_k_a        = nullptr;382    struct ggml_tensor * time_mix_r_k        = nullptr;383 384    struct ggml_tensor * time_mix_ln     = nullptr;385    struct ggml_tensor * time_mix_ln_b   = nullptr;386    struct ggml_tensor * time_mix_output = nullptr;387 388    struct ggml_tensor * channel_mix_lerp_k = nullptr;389    struct ggml_tensor * channel_mix_lerp_r = nullptr;390 391    struct ggml_tensor * channel_mix_key        = nullptr;392    struct ggml_tensor * channel_mix_receptance = nullptr;393    struct ggml_tensor * channel_mix_value      = nullptr;394 395    // long rope factors396    struct ggml_tensor * rope_long  = nullptr;397    struct ggml_tensor * rope_short = nullptr;398    struct ggml_tensor * rope_freqs = nullptr;399 400    // bitnet scale401    struct ggml_tensor * wq_s       = nullptr;402    struct ggml_tensor * wk_s       = nullptr;403    struct ggml_tensor * wv_s       = nullptr;404    struct ggml_tensor * wo_s       = nullptr;405    struct ggml_tensor * wqkv_s     = nullptr;406    struct ggml_tensor * wqkv_gate_s = nullptr;407    struct ggml_tensor * ffn_gate_s = nullptr;408    struct ggml_tensor * ffn_up_s   = nullptr;409    struct ggml_tensor * ffn_down_s = nullptr;410    struct ggml_tensor * ffn_gate_shexp_s = nullptr;411    struct ggml_tensor * ffn_up_shexp_s   = nullptr;412    struct ggml_tensor * ffn_down_shexp_s = nullptr;413    struct ggml_tensor * ssm_in_s    = nullptr;414    struct ggml_tensor * ssm_out_s   = nullptr;415    struct ggml_tensor * ssm_alpha_s = nullptr;416    struct ggml_tensor * ssm_beta_s  = nullptr;417 418    // input scales419    struct ggml_tensor * wq_in_s            = nullptr;420    struct ggml_tensor * wk_in_s            = nullptr;421    struct ggml_tensor * wv_in_s            = nullptr;422    struct ggml_tensor * wo_in_s            = nullptr;423    struct ggml_tensor * wqkv_in_s          = nullptr;424    struct ggml_tensor * wqkv_gate_in_s     = nullptr;425    struct ggml_tensor * ffn_gate_in_s      = nullptr;426    struct ggml_tensor * ffn_up_in_s        = nullptr;427    struct ggml_tensor * ffn_down_in_s      = nullptr;428    struct ggml_tensor * ffn_gate_exps_in_s = nullptr;429    struct ggml_tensor * ffn_down_exps_in_s = nullptr;430    struct ggml_tensor * ffn_up_exps_in_s   = nullptr;431    struct ggml_tensor * ffn_gate_shexp_in_s= nullptr;432    struct ggml_tensor * ffn_up_shexp_in_s  = nullptr;433    struct ggml_tensor * ffn_down_shexp_in_s= nullptr;434    struct ggml_tensor * ssm_in_in_s        = nullptr;435    struct ggml_tensor * ssm_out_in_s       = nullptr;436    struct ggml_tensor * ssm_alpha_in_s     = nullptr;437    struct ggml_tensor * ssm_beta_in_s      = nullptr;438 439    // altup & laurel440    struct ggml_tensor * per_layer_inp_gate   = nullptr;441    struct ggml_tensor * per_layer_proj       = nullptr;442    struct ggml_tensor * per_layer_post_norm  = nullptr;443    struct ggml_tensor * altup_correct_coef   = nullptr;444    struct ggml_tensor * altup_correct_scale  = nullptr;445    struct ggml_tensor * altup_predict_coef   = nullptr;446    struct ggml_tensor * altup_router         = nullptr;447    struct ggml_tensor * altup_router_norm    = nullptr;448    struct ggml_tensor * laurel_l             = nullptr;449    struct ggml_tensor * laurel_r             = nullptr;450    struct ggml_tensor * laurel_post_norm     = nullptr;451 452    // openai-moe453    struct ggml_tensor * attn_sinks = nullptr;454 455    // cogvlm456    struct ggml_tensor * visexp_attn_wqkv = nullptr;457    struct ggml_tensor * visexp_attn_wo   = nullptr;458    struct ggml_tensor * visexp_ffn_gate  = nullptr;459    struct ggml_tensor * visexp_ffn_down  = nullptr;460    struct ggml_tensor * visexp_ffn_up    = nullptr;461 462    // xIELU activation parameters for Apertus463    struct ggml_tensor * ffn_act_alpha_n = nullptr;464    struct ggml_tensor * ffn_act_alpha_p = nullptr;465    struct ggml_tensor * ffn_act_beta    = nullptr;466    struct ggml_tensor * ffn_act_eps     = nullptr;467 468    // Kimi Linear KDA (using ssm_ prefix for consistency)469    // Note: ssm_dt_b already exists above (mamba bias), reused for Kimi dt_bias470    struct ggml_tensor * ssm_q_conv = nullptr;471    struct ggml_tensor * ssm_k_conv = nullptr;472    struct ggml_tensor * ssm_v_conv = nullptr;473    struct ggml_tensor * ssm_f_a    = nullptr;474    struct ggml_tensor * ssm_f_b    = nullptr;475    struct ggml_tensor * ssm_beta   = nullptr;476    struct ggml_tensor * ssm_g_a    = nullptr;477    struct ggml_tensor * ssm_g_b    = nullptr;478    struct ggml_tensor * ssm_o_norm = nullptr;479 480    // DSA (deepseek sparse attention)481    struct ggml_tensor * indexer_k_norm   = nullptr;482    struct ggml_tensor * indexer_k_norm_b = nullptr;483    struct ggml_tensor * indexer_proj     = nullptr;484    struct ggml_tensor * indexer_attn_k   = nullptr;485    struct ggml_tensor * indexer_attn_q_b = nullptr; // note: for lora a/b, not bias486 487    // gemma4 layer output scale488    struct ggml_tensor * out_scale = nullptr;489 490    struct llama_layer_posnet posnet;491 492    struct llama_layer_convnext convnext;493 494    struct llama_layer_shortconv shortconv;495 496    struct llama_layer_nextn nextn;497};498 499struct llama_device {500    bool is_meta;501 502    ggml_backend_dev_t dev;503};504 505struct llama_meta_device_get_split_state_userdata {506    size_t                     n_devices;507    const struct llama_model * model;508};509 510struct ggml_backend_meta_split_state llama_meta_device_get_split_state(const struct ggml_tensor * tensor, void * userdata);511 512struct llama_model {513    llm_type type = LLM_TYPE_UNKNOWN;514    llm_arch arch = LLM_ARCH_UNKNOWN;515 516    std::string name = "n/a";517 518    llama_hparams hparams = {};519    llama_vocab   vocab;520 521    // for classifier models522    std::vector<std::string> classifier_labels;523 524    struct ggml_tensor * tok_embd   = nullptr;525    struct ggml_tensor * type_embd  = nullptr;526    struct ggml_tensor * pos_embd   = nullptr;527    struct ggml_tensor * tok_norm   = nullptr;528    struct ggml_tensor * tok_norm_b = nullptr;529 530    struct ggml_tensor * output_norm     = nullptr;531    struct ggml_tensor * output_norm_b   = nullptr;532    struct ggml_tensor * output          = nullptr;533    struct ggml_tensor * output_b        = nullptr;534    struct ggml_tensor * output_norm_enc = nullptr;535 536    // classifier537    struct ggml_tensor * cls       = nullptr;538    struct ggml_tensor * cls_b     = nullptr;539    struct ggml_tensor * cls_out   = nullptr;540    struct ggml_tensor * cls_out_b = nullptr;541    struct ggml_tensor * cls_norm  = nullptr;542 543    struct ggml_tensor * conv1d   = nullptr;544    struct ggml_tensor * conv1d_b = nullptr;545 546    // gemma3n altup547    struct ggml_tensor * altup_proj           = nullptr;548    struct ggml_tensor * altup_unembd_proj    = nullptr;549    struct ggml_tensor * per_layer_tok_embd   = nullptr;550    struct ggml_tensor * per_layer_model_proj = nullptr;551    struct ggml_tensor * per_layer_proj_norm  = nullptr;552 553    std::vector<llama_layer> layers;554 555    //Dense linear projections for SentenceTransformers models like embeddinggemma556    // For Sentence Transformers models structure see557    // https://sbert.net/docs/sentence_transformer/usage/custom_models.html#structure-of-sentence-transformer-models558    struct ggml_tensor * dense_2_out_layers   = nullptr;559    struct ggml_tensor * dense_2_out_layers_b = nullptr;560    struct ggml_tensor * dense_3_out_layers   = nullptr;561 562    // gguf metadata563    std::unordered_map<std::string, std::string> gguf_kv;564 565    // list of devices used in this model566    std::vector<llama_device> devices;567 568    // for quantize-stats only569    std::vector<std::pair<std::string, struct ggml_tensor *>> tensors_by_name;570 571    // for keeping track of associated LoRA adapters572    std::unordered_set<llama_adapter_lora *> loras;573 574    // statically allocated context for assigning575    struct llama_meta_device_get_split_state_userdata get_split_state_ud;576 577    int64_t t_load_us  = 0;578    int64_t t_start_us = 0;579 580    explicit llama_model(const struct llama_model_params & params);581    ~llama_model();582 583    void load_stats  (llama_model_loader & ml);584    void load_arch   (llama_model_loader & ml);585    void load_hparams(llama_model_loader & ml);586    void load_vocab  (llama_model_loader & ml);587    bool load_tensors(llama_model_loader & ml); // returns false if cancelled by progress_callback588 589    std::string arch_name() const;590    std::string type_name() const;591 592    std::string desc() const;593 594    size_t size() const; // file size595    size_t n_tensors() const;596    size_t n_devices() const;597    const float * tensor_split() const;598 599    uint32_t n_gpu_layers() const;600    llama_split_mode split_mode() const;601 602    std::map<ggml_backend_buffer_type_t, size_t> memory_breakdown() const;603 604    // total number of parameters in the model605    uint64_t n_elements() const;606 607    void print_info() const;608 609    ggml_backend_dev_t dev_layer(int il) const;610    ggml_backend_dev_t dev_output() const;611 612    ggml_backend_buffer_type_t select_buft(int il) const;613 614    bool has_tensor_overrides() const;615 616    const struct ggml_tensor * get_tensor(const char * name) const;617 618    float get_rope_freq_base (const llama_cparams & cparams, int il) const;619    float get_rope_freq_scale(const llama_cparams & cparams, int il) const;620 621    ggml_tensor * get_rope_factors(const llama_cparams & cparams, int il) const;622 623    // TODO: move this to new llm_arch_model_i interface624    llama_memory_i * create_memory(const llama_memory_params & params, const llama_cparams & cparams) const;625 626    // TODO: move this to new llm_arch_model_i interface627    ggml_cgraph * build_graph(const llm_graph_params & params) const;628 629private:630    llama_model_params params;631 632    struct impl;633    std::unique_ptr<impl> pimpl;634};635 636const char * llm_type_name(llm_type type);637 638// For internal test use639// TODO: remove640const std::vector<std::pair<std::string, ggml_tensor *>> & llama_internal_get_tensor_map(const llama_model * model);641