KBaba7/llama.cpp

1#pragma once2 3#include "ggml.h" // ggml_op4 5#include <string>6 7//8// gguf constants (sync with gguf.py)9//10 11enum llm_arch {12    LLM_ARCH_LLAMA,13    LLM_ARCH_DECI,14    LLM_ARCH_FALCON,15    LLM_ARCH_BAICHUAN,16    LLM_ARCH_GROK,17    LLM_ARCH_GPT2,18    LLM_ARCH_GPTJ,19    LLM_ARCH_GPTNEOX,20    LLM_ARCH_MPT,21    LLM_ARCH_STARCODER,22    LLM_ARCH_REFACT,23    LLM_ARCH_BERT,24    LLM_ARCH_NOMIC_BERT,25    LLM_ARCH_JINA_BERT_V2,26    LLM_ARCH_BLOOM,27    LLM_ARCH_STABLELM,28    LLM_ARCH_QWEN,29    LLM_ARCH_QWEN2,30    LLM_ARCH_QWEN2MOE,31    LLM_ARCH_QWEN2VL,32    LLM_ARCH_PHI2,33    LLM_ARCH_PHI3,34    LLM_ARCH_PHIMOE,35    LLM_ARCH_PLAMO,36    LLM_ARCH_CODESHELL,37    LLM_ARCH_ORION,38    LLM_ARCH_INTERNLM2,39    LLM_ARCH_MINICPM,40    LLM_ARCH_MINICPM3,41    LLM_ARCH_GEMMA,42    LLM_ARCH_GEMMA2,43    LLM_ARCH_STARCODER2,44    LLM_ARCH_MAMBA,45    LLM_ARCH_XVERSE,46    LLM_ARCH_COMMAND_R,47    LLM_ARCH_COHERE2,48    LLM_ARCH_DBRX,49    LLM_ARCH_OLMO,50    LLM_ARCH_OLMO2,51    LLM_ARCH_OLMOE,52    LLM_ARCH_OPENELM,53    LLM_ARCH_ARCTIC,54    LLM_ARCH_DEEPSEEK,55    LLM_ARCH_DEEPSEEK2,56    LLM_ARCH_CHATGLM,57    LLM_ARCH_BITNET,58    LLM_ARCH_T5,59    LLM_ARCH_T5ENCODER,60    LLM_ARCH_JAIS,61    LLM_ARCH_NEMOTRON,62    LLM_ARCH_EXAONE,63    LLM_ARCH_RWKV6,64    LLM_ARCH_RWKV6QWEN2,65    LLM_ARCH_GRANITE,66    LLM_ARCH_GRANITE_MOE,67    LLM_ARCH_CHAMELEON,68    LLM_ARCH_WAVTOKENIZER_DEC,69    LLM_ARCH_UNKNOWN,70};71 72enum llm_kv {73    LLM_KV_GENERAL_TYPE,74    LLM_KV_GENERAL_ARCHITECTURE,75    LLM_KV_GENERAL_QUANTIZATION_VERSION,76    LLM_KV_GENERAL_ALIGNMENT,77    LLM_KV_GENERAL_NAME,78    LLM_KV_GENERAL_AUTHOR,79    LLM_KV_GENERAL_VERSION,80    LLM_KV_GENERAL_URL,81    LLM_KV_GENERAL_DESCRIPTION,82    LLM_KV_GENERAL_LICENSE,83    LLM_KV_GENERAL_SOURCE_URL,84    LLM_KV_GENERAL_SOURCE_HF_REPO,85 86    LLM_KV_VOCAB_SIZE,87    LLM_KV_CONTEXT_LENGTH,88    LLM_KV_EMBEDDING_LENGTH,89    LLM_KV_FEATURES_LENGTH,90    LLM_KV_BLOCK_COUNT,91    LLM_KV_LEADING_DENSE_BLOCK_COUNT,92    LLM_KV_FEED_FORWARD_LENGTH,93    LLM_KV_EXPERT_FEED_FORWARD_LENGTH,94    LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH,95    LLM_KV_USE_PARALLEL_RESIDUAL,96    LLM_KV_TENSOR_DATA_LAYOUT,97    LLM_KV_EXPERT_COUNT,98    LLM_KV_EXPERT_USED_COUNT,99    LLM_KV_EXPERT_SHARED_COUNT,100    LLM_KV_EXPERT_WEIGHTS_SCALE,101    LLM_KV_EXPERT_WEIGHTS_NORM,102    LLM_KV_EXPERT_GATING_FUNC,103    LLM_KV_POOLING_TYPE,104    LLM_KV_LOGIT_SCALE,105    LLM_KV_DECODER_START_TOKEN_ID,106    LLM_KV_ATTN_LOGIT_SOFTCAPPING,107    LLM_KV_FINAL_LOGIT_SOFTCAPPING,108    LLM_KV_SWIN_NORM,109    LLM_KV_RESCALE_EVERY_N_LAYERS,110    LLM_KV_TIME_MIX_EXTRA_DIM,111    LLM_KV_TIME_DECAY_EXTRA_DIM,112    LLM_KV_RESIDUAL_SCALE,113    LLM_KV_EMBEDDING_SCALE,114    LLM_KV_TOKEN_SHIFT_COUNT,115 116    LLM_KV_ATTENTION_HEAD_COUNT,117    LLM_KV_ATTENTION_HEAD_COUNT_KV,118    LLM_KV_ATTENTION_MAX_ALIBI_BIAS,119    LLM_KV_ATTENTION_CLAMP_KQV,120    LLM_KV_ATTENTION_KEY_LENGTH,121    LLM_KV_ATTENTION_VALUE_LENGTH,122    LLM_KV_ATTENTION_LAYERNORM_EPS,123    LLM_KV_ATTENTION_LAYERNORM_RMS_EPS,124    LLM_KV_ATTENTION_GROUPNORM_EPS,125    LLM_KV_ATTENTION_GROUPNORM_GROUPS,126    LLM_KV_ATTENTION_CAUSAL,127    LLM_KV_ATTENTION_Q_LORA_RANK,128    LLM_KV_ATTENTION_KV_LORA_RANK,129    LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT,130    LLM_KV_ATTENTION_SLIDING_WINDOW,131    LLM_KV_ATTENTION_SCALE,132 133    LLM_KV_ROPE_DIMENSION_COUNT,134    LLM_KV_ROPE_DIMENSION_SECTIONS,135    LLM_KV_ROPE_FREQ_BASE,136    LLM_KV_ROPE_SCALE_LINEAR,137    LLM_KV_ROPE_SCALING_TYPE,138    LLM_KV_ROPE_SCALING_FACTOR,139    LLM_KV_ROPE_SCALING_ATTN_FACTOR,140    LLM_KV_ROPE_SCALING_ORIG_CTX_LEN,141    LLM_KV_ROPE_SCALING_FINETUNED,142    LLM_KV_ROPE_SCALING_YARN_LOG_MUL,143 144    LLM_KV_SPLIT_NO,145    LLM_KV_SPLIT_COUNT,146    LLM_KV_SPLIT_TENSORS_COUNT,147 148    LLM_KV_SSM_INNER_SIZE,149    LLM_KV_SSM_CONV_KERNEL,150    LLM_KV_SSM_STATE_SIZE,151    LLM_KV_SSM_TIME_STEP_RANK,152    LLM_KV_SSM_DT_B_C_RMS,153 154    LLM_KV_WKV_HEAD_SIZE,155 156    LLM_KV_TOKENIZER_MODEL,157    LLM_KV_TOKENIZER_PRE,158    LLM_KV_TOKENIZER_LIST,159    LLM_KV_TOKENIZER_TOKEN_TYPE,160    LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT,161    LLM_KV_TOKENIZER_SCORES,162    LLM_KV_TOKENIZER_MERGES,163    LLM_KV_TOKENIZER_BOS_ID,164    LLM_KV_TOKENIZER_EOS_ID,165    LLM_KV_TOKENIZER_EOT_ID,166    LLM_KV_TOKENIZER_EOM_ID,167    LLM_KV_TOKENIZER_UNK_ID,168    LLM_KV_TOKENIZER_SEP_ID,169    LLM_KV_TOKENIZER_PAD_ID,170    LLM_KV_TOKENIZER_CLS_ID,171    LLM_KV_TOKENIZER_MASK_ID,172    LLM_KV_TOKENIZER_ADD_BOS,173    LLM_KV_TOKENIZER_ADD_EOS,174    LLM_KV_TOKENIZER_ADD_PREFIX,175    LLM_KV_TOKENIZER_REMOVE_EXTRA_WS,176    LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP,177    LLM_KV_TOKENIZER_HF_JSON,178    LLM_KV_TOKENIZER_RWKV,179    LLM_KV_TOKENIZER_CHAT_TEMPLATE,180    LLM_KV_TOKENIZER_CHAT_TEMPLATE_N,181    LLM_KV_TOKENIZER_FIM_PRE_ID,182    LLM_KV_TOKENIZER_FIM_SUF_ID,183    LLM_KV_TOKENIZER_FIM_MID_ID,184    LLM_KV_TOKENIZER_FIM_PAD_ID,185    LLM_KV_TOKENIZER_FIM_REP_ID,186    LLM_KV_TOKENIZER_FIM_SEP_ID,187 188    LLM_KV_ADAPTER_TYPE,189    LLM_KV_ADAPTER_LORA_ALPHA,190 191    LLM_KV_POSNET_EMBEDDING_LENGTH,192    LLM_KV_POSNET_BLOCK_COUNT,193 194    LLM_KV_CONVNEXT_EMBEDDING_LENGTH,195    LLM_KV_CONVNEXT_BLOCK_COUNT,196 197    // deprecated:198    LLM_KV_TOKENIZER_PREFIX_ID,199    LLM_KV_TOKENIZER_SUFFIX_ID,200    LLM_KV_TOKENIZER_MIDDLE_ID,201};202 203enum llm_tensor {204    LLM_TENSOR_TOKEN_EMBD,205    LLM_TENSOR_TOKEN_EMBD_NORM,206    LLM_TENSOR_TOKEN_TYPES,207    LLM_TENSOR_POS_EMBD,208    LLM_TENSOR_OUTPUT,209    LLM_TENSOR_OUTPUT_NORM,210    LLM_TENSOR_ROPE_FREQS,211    LLM_TENSOR_ROPE_FACTORS_LONG,212    LLM_TENSOR_ROPE_FACTORS_SHORT,213    LLM_TENSOR_ATTN_Q,214    LLM_TENSOR_ATTN_K,215    LLM_TENSOR_ATTN_V,216    LLM_TENSOR_ATTN_QKV,217    LLM_TENSOR_ATTN_OUT,218    LLM_TENSOR_ATTN_NORM,219    LLM_TENSOR_ATTN_NORM_2,220    LLM_TENSOR_ATTN_OUT_NORM,221    LLM_TENSOR_ATTN_POST_NORM,222    LLM_TENSOR_ATTN_ROT_EMBD,223    LLM_TENSOR_FFN_GATE_INP,224    LLM_TENSOR_FFN_GATE_INP_SHEXP,225    LLM_TENSOR_FFN_NORM,226    LLM_TENSOR_FFN_POST_NORM,227    LLM_TENSOR_FFN_GATE,228    LLM_TENSOR_FFN_DOWN,229    LLM_TENSOR_FFN_UP,230    LLM_TENSOR_FFN_ACT,231    LLM_TENSOR_FFN_DOWN_EXP,  // split experts for backward compatibility232    LLM_TENSOR_FFN_GATE_EXP,233    LLM_TENSOR_FFN_UP_EXP,234    LLM_TENSOR_FFN_NORM_EXPS,235    LLM_TENSOR_FFN_DOWN_EXPS, // merged experts236    LLM_TENSOR_FFN_GATE_EXPS,237    LLM_TENSOR_FFN_UP_EXPS,238    LLM_TENSOR_FFN_DOWN_SHEXP,239    LLM_TENSOR_FFN_GATE_SHEXP,240    LLM_TENSOR_FFN_UP_SHEXP,241    LLM_TENSOR_FFN_EXP_PROBS_B,242    LLM_TENSOR_ATTN_Q_NORM,243    LLM_TENSOR_ATTN_K_NORM,244    LLM_TENSOR_LAYER_OUT_NORM,245    LLM_TENSOR_SSM_IN,246    LLM_TENSOR_SSM_CONV1D,247    LLM_TENSOR_SSM_X,248    LLM_TENSOR_SSM_DT,249    LLM_TENSOR_SSM_A,250    LLM_TENSOR_SSM_D,251    LLM_TENSOR_SSM_OUT,252    LLM_TENSOR_TIME_MIX_W1,253    LLM_TENSOR_TIME_MIX_W2,254    LLM_TENSOR_TIME_MIX_LERP_X,255    LLM_TENSOR_TIME_MIX_LERP_W,256    LLM_TENSOR_TIME_MIX_LERP_K,257    LLM_TENSOR_TIME_MIX_LERP_V,258    LLM_TENSOR_TIME_MIX_LERP_R,259    LLM_TENSOR_TIME_MIX_LERP_G,260    LLM_TENSOR_TIME_MIX_LERP_FUSED,261    LLM_TENSOR_TIME_MIX_FIRST,262    LLM_TENSOR_TIME_MIX_DECAY,263    LLM_TENSOR_TIME_MIX_DECAY_W1,264    LLM_TENSOR_TIME_MIX_DECAY_W2,265    LLM_TENSOR_TIME_MIX_KEY,266    LLM_TENSOR_TIME_MIX_VALUE,267    LLM_TENSOR_TIME_MIX_RECEPTANCE,268    LLM_TENSOR_TIME_MIX_GATE,269    LLM_TENSOR_TIME_MIX_LN,270    LLM_TENSOR_TIME_MIX_OUTPUT,271    LLM_TENSOR_CHANNEL_MIX_LERP_K,272    LLM_TENSOR_CHANNEL_MIX_LERP_R,273    LLM_TENSOR_CHANNEL_MIX_KEY,274    LLM_TENSOR_CHANNEL_MIX_RECEPTANCE,275    LLM_TENSOR_CHANNEL_MIX_VALUE,276    LLM_TENSOR_ATTN_Q_A,277    LLM_TENSOR_ATTN_Q_B,278    LLM_TENSOR_ATTN_KV_A_MQA,279    LLM_TENSOR_ATTN_KV_B,280    LLM_TENSOR_ATTN_Q_A_NORM,281    LLM_TENSOR_ATTN_KV_A_NORM,282    LLM_TENSOR_ATTN_SUB_NORM,283    LLM_TENSOR_FFN_SUB_NORM,284    LLM_TENSOR_DEC_ATTN_NORM,285    LLM_TENSOR_DEC_ATTN_Q,286    LLM_TENSOR_DEC_ATTN_K,287    LLM_TENSOR_DEC_ATTN_V,288    LLM_TENSOR_DEC_ATTN_OUT,289    LLM_TENSOR_DEC_ATTN_REL_B,290    LLM_TENSOR_DEC_CROSS_ATTN_NORM,291    LLM_TENSOR_DEC_CROSS_ATTN_Q,292    LLM_TENSOR_DEC_CROSS_ATTN_K,293    LLM_TENSOR_DEC_CROSS_ATTN_V,294    LLM_TENSOR_DEC_CROSS_ATTN_OUT,295    LLM_TENSOR_DEC_CROSS_ATTN_REL_B,296    LLM_TENSOR_DEC_FFN_NORM,297    LLM_TENSOR_DEC_FFN_GATE,298    LLM_TENSOR_DEC_FFN_DOWN,299    LLM_TENSOR_DEC_FFN_UP,300    LLM_TENSOR_DEC_OUTPUT_NORM,301    LLM_TENSOR_ENC_ATTN_NORM,302    LLM_TENSOR_ENC_ATTN_Q,303    LLM_TENSOR_ENC_ATTN_K,304    LLM_TENSOR_ENC_ATTN_V,305    LLM_TENSOR_ENC_ATTN_OUT,306    LLM_TENSOR_ENC_ATTN_REL_B,307    LLM_TENSOR_ENC_FFN_NORM,308    LLM_TENSOR_ENC_FFN_GATE,309    LLM_TENSOR_ENC_FFN_DOWN,310    LLM_TENSOR_ENC_FFN_UP,311    LLM_TENSOR_ENC_OUTPUT_NORM,312    LLM_TENSOR_CLS,313    LLM_TENSOR_CLS_OUT,314    LLM_TENSOR_CONV1D,315    LLM_TENSOR_CONVNEXT_DW,316    LLM_TENSOR_CONVNEXT_NORM,317    LLM_TENSOR_CONVNEXT_PW1,318    LLM_TENSOR_CONVNEXT_PW2,319    LLM_TENSOR_CONVNEXT_GAMMA,320    LLM_TENSOR_POS_NET_CONV1,321    LLM_TENSOR_POS_NET_CONV2,322    LLM_TENSOR_POS_NET_NORM,323    LLM_TENSOR_POS_NET_NORM1,324    LLM_TENSOR_POS_NET_NORM2,325    LLM_TENSOR_POS_NET_ATTN_NORM,326    LLM_TENSOR_POS_NET_ATTN_Q,327    LLM_TENSOR_POS_NET_ATTN_K,328    LLM_TENSOR_POS_NET_ATTN_V,329    LLM_TENSOR_POS_NET_ATTN_OUT,330};331 332enum llm_tensor_layer {333    LLM_TENSOR_LAYER_INPUT,334    LLM_TENSOR_LAYER_REPEATING,335    LLM_TENSOR_LAYER_OUTPUT,336};337 338struct LLM_KV {339    LLM_KV(llm_arch arch, const char * suffix = nullptr);340 341    llm_arch arch;342    const char * suffix;343 344    std::string operator()(llm_kv kv) const;345};346 347// helper to handle gguf constants348// usage:349//350//   const auto tn = LLM_TN(LLM_ARCH_LLAMA);351//352//   std::string name = tn(LLM_TENSOR_OUTPUT);                     -> "output"353//   std::string name = tn(LLM_TENSOR_TOKEN_EMBD, "bias");         -> "token_embd.bias"354//   std::string name = tn(LLM_TENSOR_ATTN_NORM, "weight", 3);     -> "blk.3.attn_norm.weight"355//356struct LLM_TN_IMPL {357    const llm_arch arch;358    const llm_tensor tensor;359    const char * const suffix;360    const int bid;361    const int xid;362 363    std::string str() const;364 365    operator std::string() const {366        return str();367    }368 369    friend bool operator==(const std::string & str, const LLM_TN_IMPL & tn) {370        return str == tn.str();371    }372 373    friend bool operator!=(const std::string & str, const LLM_TN_IMPL & tn) {374        return str != tn.str();375    }376};377 378struct LLM_TN {379    LLM_TN(llm_arch arch) : arch(arch) {}380 381    llm_arch arch;382 383    LLM_TN_IMPL operator()(llm_tensor tensor, const char * suffix, int bid = -1, int xid = -1) const {384        return { arch, tensor, suffix, bid, xid };385    }386 387    LLM_TN_IMPL operator()(llm_tensor tensor, int bid = -1, int xid = -1) const {388        return { arch, tensor, nullptr, bid, xid };389    }390};391 392 393struct llm_tensor_info {394    llm_tensor_layer layer;395    ggml_op op;396};397 398const char * llm_arch_name(llm_arch arch);399 400llm_arch llm_arch_from_string(const std::string & name);401 402const llm_tensor_info & llm_tensor_info_for(llm_tensor tensor);403