echodict/llama.cpp

1#pragma once2 3#include "llama.h"4 5#include <array>6#include <cassert>7 8// bump if necessary9#define LLAMA_MAX_LAYERS  51210#define LLAMA_MAX_EXPERTS 512 // Qwen3 Next11 12enum llama_expert_gating_func_type {13    LLAMA_EXPERT_GATING_FUNC_TYPE_NONE           = 0,14    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX        = 1,15    LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID        = 2,16    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX_WEIGHT = 3, // applied to the router weights instead of the logits17};18 19enum llama_swa_type {20    LLAMA_SWA_TYPE_NONE      = 0,21    LLAMA_SWA_TYPE_STANDARD  = 1,22    LLAMA_SWA_TYPE_CHUNKED   = 2,23    LLAMA_SWA_TYPE_SYMMETRIC = 3,24};25 26struct llama_hparams_posnet {27    uint32_t n_embd;28    uint32_t n_layer;29};30 31struct llama_hparams_convnext {32    uint32_t n_embd;33    uint32_t n_layer;34};35 36struct llama_hparams {37    bool vocab_only;38    bool no_alloc;39    bool rope_finetuned;40    bool use_par_res;41    bool swin_norm;42 43    uint32_t n_ctx_train; // context size the model was trained on44    uint32_t n_embd;45    uint32_t n_layer;46    int32_t n_layer_kv_from_start = -1; // if non-negative, the first n_layer_kv_from_start layers have KV cache47    uint32_t n_expert = 0;48    uint32_t n_expert_used = 0;49    uint32_t n_rel_attn_bkts = 0;50 51    // different head size for full_attention and SWA layers52    uint32_t n_embd_head_k_full; // dimension of keys (d_k). d_q is assumed to be the same, but there are n_head q heads, and only n_head_kv k-v heads53    uint32_t n_embd_head_v_full; // dimension of values (d_v) aka n_embd_head54    uint32_t n_embd_head_k_swa;55    uint32_t n_embd_head_v_swa;56 57    // different RoPE dimensions for full_attention and SWA layers58    uint32_t n_rot_full;59    uint32_t n_rot_swa;60 61    // note: deepseek2 using MLA converts into MQA with larger heads, then decompresses to MHA62    uint32_t n_embd_head_k_mla_impl = 0;63    uint32_t n_embd_head_v_mla_impl = 0;64 65    // for WavTokenizer66    struct llama_hparams_posnet   posnet;67    struct llama_hparams_convnext convnext;68 69    uint32_t n_shortconv_l_cache  = 0;70 71    std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_arr;72    std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_kv_arr;73    std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_arr;74 75    uint32_t n_layer_dense_lead = 0;76    uint32_t n_lora_q           = 0;77    uint32_t n_lora_kv          = 0;78    uint32_t n_ff_exp           = 0;79    uint32_t n_ff_shexp         = 0;80    uint32_t n_ff_chexp         = 0;81    uint32_t n_expert_shared    = 0;82    uint32_t n_norm_groups      = 0;83    uint32_t n_expert_groups    = 0;84    uint32_t n_group_used       = 0;85    uint32_t n_group_experts    = 0;86 87    float    expert_group_scale   = 0.05f;88    float    expert_weights_scale = 0.0f;89    bool     expert_weights_norm  = false;90    uint32_t expert_gating_func   = LLAMA_EXPERT_GATING_FUNC_TYPE_NONE;91    uint32_t moe_every_n_layers   = 0;92    uint32_t moe_latent_size      = 0;93    uint32_t nextn_predict_layers = 0;94 95    float f_norm_eps;96    float f_norm_rms_eps;97    float f_norm_group_eps;98 99    float f_attn_logit_softcapping   = 50.0f;100    float f_router_logit_softcapping = 30.0f;101    float f_final_logit_softcapping  = 30.0f;102 103    // for RWKV104    uint32_t rescale_every_n_layers = 0;105    uint32_t time_mix_extra_dim     = 0;106    uint32_t time_decay_extra_dim   = 0;107    uint32_t wkv_head_size          = 0;108    uint32_t token_shift_count      = 2;109    uint32_t n_lora_decay           = 0;110    uint32_t n_lora_iclr            = 0;111    uint32_t n_lora_value_res_mix   = 0;112    uint32_t n_lora_gate            = 0;113 114    float    rope_attn_factor = 1.0f;115    float    rope_freq_base_train;116    float    rope_freq_base_train_swa  = 10000.0f;117    float    rope_freq_scale_train;118    float    rope_freq_scale_train_swa = 1.0f;119 120    uint32_t n_ctx_orig_yarn;121    float    rope_yarn_log_mul = 0.0f;122 123    float    yarn_ext_factor  = -1.0f;124    float    yarn_attn_factor =  1.0f;125    float    yarn_beta_fast   = 32.0f;126    float    yarn_beta_slow   =  1.0f;127 128    std::array<int, 4> rope_sections;129 130    // Sliding Window Attention (SWA)131    llama_swa_type swa_type = LLAMA_SWA_TYPE_NONE;132    // the size of the sliding window (0 - no SWA)133    uint32_t n_swa = 0;134    // if swa_layers[il] == 1, then layer il is SWA135    // if swa_layers[il] == 0, then layer il is dense (i.e. non-SWA)136    // by default, all layers are dense137    // note: using uint32_t type for compatibility reason138    std::array<uint32_t, LLAMA_MAX_LAYERS> swa_layers;139 140    // for State Space Models141    uint32_t ssm_d_conv  = 0;142    uint32_t ssm_d_inner = 0;143    uint32_t ssm_d_state = 0;144    uint32_t ssm_dt_rank = 0;145    uint32_t ssm_n_group = 0;146 147    // for Kimi Linear KDA148    uint32_t n_embd_head_kda = 0;149 150    // for hybrid state space models151    std::array<bool, LLAMA_MAX_LAYERS> recurrent_layer_arr;152 153    bool ssm_dt_b_c_rms = false;154 155    float f_clamp_kqv      = 0.0f;156    float f_max_alibi_bias = 0.0f;157    float f_logit_scale    = 0.0f;158 159    // Additional scale factors (Granite/Granite MoE)160    float f_residual_scale  = 0.0f;161    float f_embedding_scale = 0.0f;162    float f_attention_scale = 0.0f;163 164    // grok-2165    float    f_attn_out_scale = 0.0f;166    uint32_t attn_temp_length = 0;167 168    bool causal_attn   = true;169    bool use_alibi     = false;170    bool attn_soft_cap = false;171    bool use_kq_norm   = false;172 173    // for Classifiers174    uint32_t n_cls_out = 1;175 176    // output embedding dimension (0 = use n_embd)177    uint32_t n_embd_out_impl = 0;178 179    // llama4 smallthinker180    uint32_t n_moe_layer_step        = 0;181    uint32_t n_no_rope_layer_step    = 4;182    uint32_t n_attn_temp_floor_scale = 0;183    float    f_attn_temp_scale       = 0.0f;184    float    f_attn_temp_offset      = 0.0f; // offset position index185 186    // gemma3n altup187    uint32_t n_altup      = 4; // altup_num_inputs188    uint32_t i_altup_act  = 0; // altup_active_idx189    uint32_t laurel_rank  = 64;190    uint32_t n_embd_altup = 256;191 192    // needed for sentence-transformers dense layers193    uint32_t dense_2_feat_in  = 0;  // in_features of the 2_Dense194    uint32_t dense_2_feat_out = 0;  // out_features of the 2_Dense195    uint32_t dense_3_feat_in  = 0;  // in_features of the 3_Dense196    uint32_t dense_3_feat_out = 0;  // out_features of the 3_Dense197 198    // xIELU199    std::array<float, LLAMA_MAX_LAYERS> xielu_alpha_n;200    std::array<float, LLAMA_MAX_LAYERS> xielu_alpha_p;201    std::array<float, LLAMA_MAX_LAYERS> xielu_beta;202    std::array<float, LLAMA_MAX_LAYERS> xielu_eps;203 204    // DSA (deepseek sparse attention)205    uint32_t indexer_n_head    = 0;206    uint32_t indexer_head_size = 0;207    uint32_t indexer_top_k     = 0;208 209    // qwen3vl deepstack210    uint32_t n_deepstack_layers = 0;211 212    // gemma4 per-layer embedding213    uint32_t n_embd_per_layer = 0;214 215    // needed by encoder-decoder models (e.g. T5, FLAN-T5)216    // ref: https://github.com/ggml-org/llama.cpp/pull/8141217    llama_token dec_start_token_id = LLAMA_TOKEN_NULL;218    uint32_t    dec_n_layer        = 0;219 220    enum llama_pooling_type      pooling_type            = LLAMA_POOLING_TYPE_NONE;221    enum llama_rope_type         rope_type               = LLAMA_ROPE_TYPE_NONE;222    enum llama_rope_scaling_type rope_scaling_type_train = LLAMA_ROPE_SCALING_TYPE_NONE;223 224 225    // Step35: optional per-layer clamps for (Swi)GLU226    std::array<float, LLAMA_MAX_LAYERS> swiglu_clamp_exp; // clamping for expert FFN227    std::array<float, LLAMA_MAX_LAYERS> swiglu_clamp_shexp; // shared expert228 229    // this value n_pattern means that every nth layer is dense (i.e. non-SWA)230    // dense_first means whether the pattern is start with a dense layer231    // note that if n_pattern == 0, all layers are SWA232    //           if n_pattern == 1, all layers are dense233    // example 1: n_pattern = 3, dense_first = false234    //   il == 0: swa235    //   il == 1: swa236    //   il == 2: dense237    //   il == 3: swa238    //   il == 4: swa239    //   il == 5: dense240    //   il == 6: swa241    //   etc ...242    // example 2: n_pattern = 2, dense_first = true243    //   il == 0: dense244    //   il == 1: swa245    //   il == 2: dense246    //   il == 3: swa247    //   etc ...248    void set_swa_pattern(uint32_t n_pattern, bool dense_first = false);249 250    // return true if one of the layers is SWA251    bool is_swa_any() const;252 253    uint32_t n_head(uint32_t il = 0) const;254 255    uint32_t n_head_kv(uint32_t il = 0) const;256 257    uint32_t n_ff(uint32_t il = 0) const;258 259    uint32_t n_gqa(uint32_t il = 0) const;260 261    uint32_t n_rot(uint32_t il = 0) const;262 263    // dimension of main + auxiliary input embeddings264    uint32_t n_embd_inp() const;265 266    // dimension of output embeddings267    uint32_t n_embd_out() const;268 269    // dimension of key/value embeddings for each head (per layer)270    uint32_t n_embd_head_k(uint32_t il = 0) const;271    uint32_t n_embd_head_v(uint32_t il = 0) const;272 273    // dimension of key embeddings across all k-v heads274    uint32_t n_embd_k_gqa(uint32_t il = 0) const;275 276    // dimension of value embeddings across all k-v heads277    uint32_t n_embd_v_gqa(uint32_t il = 0) const;278 279    // true if any layer has a different n_embd_k_gqa/n_embd_v_gqa280    bool is_n_embd_k_gqa_variable() const;281    bool is_n_embd_v_gqa_variable() const;282 283    // return the maximum n_embd_k_gqa/n_embd_v_gqa across all layers284    uint32_t n_embd_k_gqa_max() const;285    uint32_t n_embd_v_gqa_max() const;286 287    // dimension of the rolling state embeddings288    // corresponds to Mamba's conv_states size or RWKV's token_shift states size289    uint32_t n_embd_r() const;290 291    // dimension of the recurrent state embeddings292    uint32_t n_embd_s() const;293 294    // whether or not the given layer is recurrent (for hybrid models)295    bool is_recurrent(uint32_t il) const;296 297    uint32_t n_pos_per_embd() const;298 299    bool is_swa(uint32_t il) const;300 301    // note: currently only support if either all or none of the layers are MLA302    bool is_mla() const;303 304    uint32_t n_embd_head_k_mla() const;305    uint32_t n_embd_head_v_mla() const;306 307    bool has_kv(uint32_t il) const;308 309    // number of layers for which has_kv() returns true310    uint32_t n_layer_kv() const;311 312    // note that this function uses different SWA parameters from those in the hparams313    // note: inlined on purpose for performance reasons314    // TODO: think of a better place for this function315    // TODO: pack the SWA params in a struct?316    static bool is_masked_swa(uint32_t n_swa, llama_swa_type swa_type, llama_pos p0, llama_pos p1) {317        assert(p0 >= 0 && p1 >= 0);318 319        switch (swa_type) {320            case LLAMA_SWA_TYPE_NONE:321                {322                } break;323            case LLAMA_SWA_TYPE_STANDARD:324                {325                    if (p1 - p0 >= (int32_t) n_swa) {326                        return true;327                    }328                } break;329            case LLAMA_SWA_TYPE_CHUNKED:330                {331                    const llama_pos pos_chunk_start = (p1 / n_swa) * n_swa;332 333                    if (p0 < pos_chunk_start) {334                        return true;335                    }336                } break;337            case LLAMA_SWA_TYPE_SYMMETRIC:338                {339                    const int32_t half_n_swa = (int32_t) n_swa / 2;340                    const int32_t pos_diff = p1 - p0;341 342                    // Mask if outside the symmetric window343                    if (pos_diff < -half_n_swa || pos_diff > half_n_swa) {344                        return true;345                    }346                } break;347        }348 349        return false;350    }351 352 353    bool use_mrope() const;354};355 356static_assert(std::is_trivially_copyable<llama_hparams>::value, "llama_hparams must be trivially copyable");357