KBaba7/llama.cpp

1#pragma once2 3#include "llama.h"4#include "llama-arch.h"5#include "llama-hparams.h"6#include "llama-vocab.h"7 8#include <memory>9#include <string>10#include <unordered_map>11#include <vector>12 13struct llama_model_loader;14 15// available models16enum llm_type {17    LLM_TYPE_UNKNOWN,18    LLM_TYPE_14M,19    LLM_TYPE_17M,20    LLM_TYPE_22M,21    LLM_TYPE_33M,22    LLM_TYPE_60M,23    LLM_TYPE_70M,24    LLM_TYPE_80M,25    LLM_TYPE_109M,26    LLM_TYPE_137M,27    LLM_TYPE_160M,28    LLM_TYPE_220M,29    LLM_TYPE_250M,30    LLM_TYPE_270M,31    LLM_TYPE_335M,32    LLM_TYPE_410M,33    LLM_TYPE_450M,34    LLM_TYPE_770M,35    LLM_TYPE_780M,36    LLM_TYPE_0_5B,37    LLM_TYPE_1B,38    LLM_TYPE_1_3B,39    LLM_TYPE_1_4B,40    LLM_TYPE_1_5B,41    LLM_TYPE_1_6B,42    LLM_TYPE_2B,43    LLM_TYPE_2_8B,44    LLM_TYPE_3B,45    LLM_TYPE_4B,46    LLM_TYPE_6B,47    LLM_TYPE_6_9B,48    LLM_TYPE_7B,49    LLM_TYPE_8B,50    LLM_TYPE_9B,51    LLM_TYPE_11B,52    LLM_TYPE_12B,53    LLM_TYPE_13B,54    LLM_TYPE_14B,55    LLM_TYPE_15B,56    LLM_TYPE_16B,57    LLM_TYPE_20B,58    LLM_TYPE_30B,59    LLM_TYPE_32B,60    LLM_TYPE_34B,61    LLM_TYPE_35B,62    LLM_TYPE_40B,63    LLM_TYPE_65B,64    LLM_TYPE_70B,65    LLM_TYPE_236B,66    LLM_TYPE_314B,67    LLM_TYPE_671B,68    LLM_TYPE_SMALL,69    LLM_TYPE_MEDIUM,70    LLM_TYPE_LARGE,71    LLM_TYPE_XL,72    LLM_TYPE_A1_7B,73    LLM_TYPE_A2_7B,74    LLM_TYPE_8x7B,75    LLM_TYPE_8x22B,76    LLM_TYPE_16x12B,77    LLM_TYPE_16x3_8B,78    LLM_TYPE_10B_128x3_66B,79    LLM_TYPE_57B_A14B,80    LLM_TYPE_27B,81};82 83struct llama_layer_posnet {84    // resnet85    struct ggml_tensor * norm1   = nullptr;86    struct ggml_tensor * norm1_b = nullptr;87 88    struct ggml_tensor * conv1   = nullptr;89    struct ggml_tensor * conv1_b = nullptr;90 91    struct ggml_tensor * norm2   = nullptr;92    struct ggml_tensor * norm2_b = nullptr;93 94    struct ggml_tensor * conv2   = nullptr;95    struct ggml_tensor * conv2_b = nullptr;96 97    // attention98    struct ggml_tensor * attn_norm   = nullptr;99    struct ggml_tensor * attn_norm_b = nullptr;100 101    struct ggml_tensor * attn_q   = nullptr;102    struct ggml_tensor * attn_q_b = nullptr;103 104    struct ggml_tensor * attn_k   = nullptr;105    struct ggml_tensor * attn_k_b = nullptr;106 107    struct ggml_tensor * attn_v   = nullptr;108    struct ggml_tensor * attn_v_b = nullptr;109 110    struct ggml_tensor * attn_o   = nullptr;111    struct ggml_tensor * attn_o_b = nullptr;112 113    // normalize114    struct ggml_tensor * norm   = nullptr;115    struct ggml_tensor * norm_b = nullptr;116};117 118struct llama_layer_convnext {119    struct ggml_tensor * dw   = nullptr;120    struct ggml_tensor * dw_b = nullptr;121 122    struct ggml_tensor * norm   = nullptr;123    struct ggml_tensor * norm_b = nullptr;124 125    struct ggml_tensor * pw1   = nullptr;126    struct ggml_tensor * pw1_b = nullptr;127 128    struct ggml_tensor * pw2   = nullptr;129    struct ggml_tensor * pw2_b = nullptr;130 131    struct ggml_tensor * gamma = nullptr;132};133 134struct llama_layer {135    // normalization136    struct ggml_tensor * attn_norm       = nullptr;137    struct ggml_tensor * attn_norm_b     = nullptr;138    struct ggml_tensor * attn_norm_2     = nullptr;139    struct ggml_tensor * attn_norm_2_b   = nullptr;140    struct ggml_tensor * attn_q_norm     = nullptr;141    struct ggml_tensor * attn_q_norm_b   = nullptr;142    struct ggml_tensor * attn_k_norm     = nullptr;143    struct ggml_tensor * attn_k_norm_b   = nullptr;144    struct ggml_tensor * attn_out_norm   = nullptr;145    struct ggml_tensor * attn_out_norm_b = nullptr;146    struct ggml_tensor * attn_q_a_norm   = nullptr;147    struct ggml_tensor * attn_kv_a_norm  = nullptr;148    struct ggml_tensor * attn_sub_norm   = nullptr;149    struct ggml_tensor * attn_post_norm  = nullptr;150    struct ggml_tensor * ffn_sub_norm    = nullptr;151    struct ggml_tensor * attn_norm_cross = nullptr;152    struct ggml_tensor * attn_norm_enc   = nullptr;153 154    // attention155    struct ggml_tensor * wq        = nullptr;156    struct ggml_tensor * wk        = nullptr;157    struct ggml_tensor * wv        = nullptr;158    struct ggml_tensor * wo        = nullptr;159    struct ggml_tensor * wqkv      = nullptr;160    struct ggml_tensor * wq_a      = nullptr;161    struct ggml_tensor * wq_b      = nullptr;162    struct ggml_tensor * wkv_a_mqa = nullptr;163    struct ggml_tensor * wkv_b     = nullptr;164    struct ggml_tensor * wq_cross  = nullptr;165    struct ggml_tensor * wk_cross  = nullptr;166    struct ggml_tensor * wv_cross  = nullptr;167    struct ggml_tensor * wo_cross  = nullptr;168    struct ggml_tensor * wq_enc    = nullptr;169    struct ggml_tensor * wk_enc    = nullptr;170    struct ggml_tensor * wv_enc    = nullptr;171    struct ggml_tensor * wo_enc    = nullptr;172 173    // attention bias174    struct ggml_tensor * bq   = nullptr;175    struct ggml_tensor * bk   = nullptr;176    struct ggml_tensor * bv   = nullptr;177    struct ggml_tensor * bo   = nullptr;178    struct ggml_tensor * bqkv = nullptr;179 180    // relative position bias181    struct ggml_tensor * attn_rel_b       = nullptr;182    struct ggml_tensor * attn_rel_b_enc   = nullptr;183    struct ggml_tensor * attn_rel_b_cross = nullptr;184 185    // normalization186    struct ggml_tensor * ffn_norm         = nullptr;187    struct ggml_tensor * ffn_norm_b       = nullptr;188    struct ggml_tensor * ffn_post_norm    = nullptr;189    struct ggml_tensor * layer_out_norm   = nullptr;190    struct ggml_tensor * layer_out_norm_b = nullptr;191    struct ggml_tensor * ffn_norm_exps    = nullptr;192    struct ggml_tensor * ffn_norm_enc     = nullptr;193 194    // ff195    struct ggml_tensor * ffn_gate     = nullptr; // w1196    struct ggml_tensor * ffn_down     = nullptr; // w2197    struct ggml_tensor * ffn_up       = nullptr; // w3198    struct ggml_tensor * ffn_gate_enc = nullptr;199    struct ggml_tensor * ffn_down_enc = nullptr;200    struct ggml_tensor * ffn_up_enc   = nullptr;201 202    // ff MoE203    struct ggml_tensor * ffn_gate_inp  = nullptr;204    struct ggml_tensor * ffn_gate_exps = nullptr;205    struct ggml_tensor * ffn_down_exps = nullptr;206    struct ggml_tensor * ffn_up_exps   = nullptr;207 208    // ff shared expert (shexp)209    struct ggml_tensor * ffn_gate_inp_shexp = nullptr;210    struct ggml_tensor * ffn_gate_shexp     = nullptr;211    struct ggml_tensor * ffn_down_shexp     = nullptr;212    struct ggml_tensor * ffn_up_shexp       = nullptr;213 214    // ff bias215    struct ggml_tensor * ffn_gate_b = nullptr;216    struct ggml_tensor * ffn_down_b = nullptr; // b2217    struct ggml_tensor * ffn_up_b   = nullptr; // b3218    struct ggml_tensor * ffn_act    = nullptr;219    struct ggml_tensor * ffn_exp_probs_b = nullptr;220 221    // mamba proj222    struct ggml_tensor * ssm_in  = nullptr;223    struct ggml_tensor * ssm_x   = nullptr;224    struct ggml_tensor * ssm_dt  = nullptr;225    struct ggml_tensor * ssm_out = nullptr;226 227    // mamba228    struct ggml_tensor * ssm_conv1d = nullptr;229    struct ggml_tensor * ssm_a      = nullptr;230    struct ggml_tensor * ssm_d      = nullptr;231 232    // mamba bias233    struct ggml_tensor * ssm_conv1d_b = nullptr;234    struct ggml_tensor * ssm_dt_b     = nullptr;235 236    // rwkv237    struct ggml_tensor * time_mix_w1         = nullptr;238    struct ggml_tensor * time_mix_w2         = nullptr;239    struct ggml_tensor * time_mix_lerp_x     = nullptr;240    struct ggml_tensor * time_mix_lerp_w     = nullptr;241    struct ggml_tensor * time_mix_lerp_k     = nullptr;242    struct ggml_tensor * time_mix_lerp_v     = nullptr;243    struct ggml_tensor * time_mix_lerp_r     = nullptr;244    struct ggml_tensor * time_mix_lerp_g     = nullptr;245    struct ggml_tensor * time_mix_lerp_fused = nullptr;246 247    struct ggml_tensor * time_mix_first        = nullptr;248    struct ggml_tensor * time_mix_decay        = nullptr;249    struct ggml_tensor * time_mix_decay_w1     = nullptr;250    struct ggml_tensor * time_mix_decay_w2     = nullptr;251    struct ggml_tensor * time_mix_key          = nullptr;252    struct ggml_tensor * time_mix_key_b        = nullptr;253    struct ggml_tensor * time_mix_value        = nullptr;254    struct ggml_tensor * time_mix_value_b      = nullptr;255    struct ggml_tensor * time_mix_receptance   = nullptr;256    struct ggml_tensor * time_mix_receptance_b = nullptr;257    struct ggml_tensor * time_mix_gate         = nullptr;258 259    struct ggml_tensor * time_mix_ln     = nullptr;260    struct ggml_tensor * time_mix_ln_b   = nullptr;261    struct ggml_tensor * time_mix_output = nullptr;262 263    struct ggml_tensor * channel_mix_lerp_k = nullptr;264    struct ggml_tensor * channel_mix_lerp_r = nullptr;265 266    struct ggml_tensor * channel_mix_key        = nullptr;267    struct ggml_tensor * channel_mix_receptance = nullptr;268    struct ggml_tensor * channel_mix_value      = nullptr;269 270    // long rope factors271    struct ggml_tensor * rope_long  = nullptr;272    struct ggml_tensor * rope_short = nullptr;273    struct ggml_tensor * rope_freqs = nullptr;274 275    // bitnet scale276    struct ggml_tensor * wq_scale       = nullptr;277    struct ggml_tensor * wk_scale       = nullptr;278    struct ggml_tensor * wv_scale       = nullptr;279    struct ggml_tensor * wo_scale       = nullptr;280    struct ggml_tensor * ffn_gate_scale = nullptr;281    struct ggml_tensor * ffn_up_scale   = nullptr;282    struct ggml_tensor * ffn_down_scale = nullptr;283 284    struct llama_layer_posnet posnet;285 286    struct llama_layer_convnext convnext;287};288 289struct llama_model {290    llm_type type = LLM_TYPE_UNKNOWN;291    llm_arch arch = LLM_ARCH_UNKNOWN;292 293    std::string name = "n/a";294 295    llama_hparams hparams = {};296    llama_vocab   vocab;297 298    struct ggml_tensor * tok_embd   = nullptr;299    struct ggml_tensor * type_embd  = nullptr;300    struct ggml_tensor * pos_embd   = nullptr;301    struct ggml_tensor * tok_norm   = nullptr;302    struct ggml_tensor * tok_norm_b = nullptr;303 304    struct ggml_tensor * output_norm     = nullptr;305    struct ggml_tensor * output_norm_b   = nullptr;306    struct ggml_tensor * output          = nullptr;307    struct ggml_tensor * output_b        = nullptr;308    struct ggml_tensor * output_norm_enc = nullptr;309 310    // classifier311    struct ggml_tensor * cls       = nullptr;312    struct ggml_tensor * cls_b     = nullptr;313    struct ggml_tensor * cls_out   = nullptr;314    struct ggml_tensor * cls_out_b = nullptr;315 316    struct ggml_tensor * conv1d   = nullptr;317    struct ggml_tensor * conv1d_b = nullptr;318 319    std::vector<llama_layer> layers;320 321    llama_model_params params;322 323    // gguf metadata324    std::unordered_map<std::string, std::string> gguf_kv;325 326    // list of devices used in this model327    std::vector<ggml_backend_dev_t> devices;328 329    // for quantize-stats only330    std::vector<std::pair<std::string, struct ggml_tensor *>> tensors_by_name;331 332    int64_t t_load_us  = 0;333    int64_t t_start_us = 0;334 335    explicit llama_model(const struct llama_model_params & params);336    ~llama_model();337 338    void load_stats  (llama_model_loader & ml);339    void load_arch   (llama_model_loader & ml);340    void load_hparams(llama_model_loader & ml);341    void load_vocab  (llama_model_loader & ml);342    bool load_tensors(llama_model_loader & ml); // returns false if cancelled by progress_callback343 344    std::string arch_name() const;345    std::string type_name() const;346 347    std::string desc() const;348 349    size_t size() const;350    size_t max_nodes() const;351    size_t n_devices() const;352 353    // total number of parameters in the model354    uint64_t n_elements() const;355 356    void print_info() const;357 358    ggml_backend_dev_t dev_layer(int il) const;359    ggml_backend_dev_t dev_output() const;360 361    ggml_backend_buffer_type_t select_buft(int il) const;362 363    const struct ggml_tensor * get_tensor(const char * name) const;364 365private:366    struct impl;367    std::unique_ptr<impl> pimpl;368};369 370const char * llm_type_name(llm_type type);371