KBaba7/llama.cpp

1#pragma once2 3#include "llama.h"4 5#include <array>6 7// bump if necessary8#define LLAMA_MAX_LAYERS  5129#define LLAMA_MAX_EXPERTS 256  // DeepSeekV310 11enum llama_expert_gating_func_type {12    LLAMA_EXPERT_GATING_FUNC_TYPE_NONE    = 0,13    LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX = 1,14    LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID = 2,15};16 17struct llama_hparams_posnet {18    uint32_t n_embd;19    uint32_t n_layer;20};21 22struct llama_hparams_convnext {23    uint32_t n_embd;24    uint32_t n_layer;25};26 27struct llama_hparams {28    bool vocab_only;29    bool rope_finetuned;30    bool use_par_res;31    bool swin_norm;32 33    uint32_t n_ctx_train; // context size the model was trained on34    uint32_t n_embd;35    uint32_t n_embd_features = 0;36    uint32_t n_layer;37    uint32_t n_rot;38    uint32_t n_swa = 0; // sliding window attention (SWA)39    uint32_t n_embd_head_k; // dimension of keys (d_k). d_q is assumed to be the same, but there are n_head q heads, and only n_head_kv k-v heads40    uint32_t n_embd_head_v; // dimension of values (d_v) aka n_embd_head41    uint32_t n_expert = 0;42    uint32_t n_expert_used = 0;43    uint32_t n_rel_attn_bkts = 0;44 45    // for WavTokenizer46    struct llama_hparams_posnet   posnet;47    struct llama_hparams_convnext convnext;48 49    std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_arr;50    std::array<uint32_t, LLAMA_MAX_LAYERS> n_head_kv_arr;51    std::array<uint32_t, LLAMA_MAX_LAYERS> n_ff_arr;52 53    uint32_t n_layer_dense_lead = 0;54    uint32_t n_lora_q           = 0;55    uint32_t n_lora_kv          = 0;56    uint32_t n_ff_exp           = 0;57    uint32_t n_ff_shexp         = 0;58    uint32_t n_expert_shared    = 0;59    uint32_t n_norm_groups      = 0;60 61    float    expert_weights_scale = 0.0;62    bool     expert_weights_norm  = false;63    uint32_t expert_gating_func   = LLAMA_EXPERT_GATING_FUNC_TYPE_NONE;64 65    float f_norm_eps;66    float f_norm_rms_eps;67    float f_norm_group_eps;68 69    float f_attn_logit_softcapping  = 50.0f;70    float f_final_logit_softcapping = 30.0f;71 72    // for RWKV73    uint32_t rescale_every_n_layers = 0;74    uint32_t time_mix_extra_dim     = 0;75    uint32_t time_decay_extra_dim   = 0;76    uint32_t wkv_head_size          = 0;77    uint32_t token_shift_count      = 2;78 79    float    rope_attn_factor = 1.0f;80    float    rope_freq_base_train;81    float    rope_freq_scale_train;82    uint32_t n_ctx_orig_yarn;83    float    rope_yarn_log_mul;84 85    std::array<int, 4> rope_sections;86 87    // for State Space Models88    uint32_t ssm_d_conv  = 0;89    uint32_t ssm_d_inner = 0;90    uint32_t ssm_d_state = 0;91    uint32_t ssm_dt_rank = 0;92 93    bool ssm_dt_b_c_rms = false;94 95    float f_clamp_kqv      = 0.0f;96    float f_max_alibi_bias = 0.0f;97    float f_logit_scale    = 0.0f;98 99    // Additional scale factors (Granite/Granite MoE)100    float f_residual_scale  = 0.0f;101    float f_embedding_scale = 0.0f;102    float f_attention_scale = 0.0f;103 104    bool causal_attn   = true;105    bool use_alibi     = false;106    bool attn_soft_cap = false;107 108    // needed by encoder-decoder models (e.g. T5, FLAN-T5)109    // ref: https://github.com/ggerganov/llama.cpp/pull/8141110    llama_token dec_start_token_id = LLAMA_TOKEN_NULL;111 112    enum llama_pooling_type      pooling_type            = LLAMA_POOLING_TYPE_NONE;113    enum llama_rope_type         rope_type               = LLAMA_ROPE_TYPE_NONE;114    enum llama_rope_scaling_type rope_scaling_type_train = LLAMA_ROPE_SCALING_TYPE_NONE;115 116    uint32_t n_head(uint32_t il = 0) const;117 118    uint32_t n_head_kv(uint32_t il = 0) const;119 120    uint32_t n_ff(uint32_t il = 0) const;121 122    uint32_t n_gqa(uint32_t il = 0) const;123 124    // dimension of key embeddings across all k-v heads125    uint32_t n_embd_k_gqa(uint32_t il = 0) const;126 127    // dimension of value embeddings across all k-v heads128    uint32_t n_embd_v_gqa(uint32_t il = 0) const;129 130    // dimension of the rolling state embeddings131    // corresponds to Mamba's conv_states size or RWKV's token_shift states size132    uint32_t n_embd_k_s() const;133 134    // dimension of the recurrent state embeddings135    uint32_t n_embd_v_s() const;136};137 138static_assert(std::is_trivially_copyable<llama_hparams>::value, "llama_hparams must be trivially copyable");139 140