echodict/llama.cpp
version https://git-lfs.github.com/spec/v1 oid sha256:cfc44b7ba25614df70e6b65e3341cae0310163bd32fd31a6b928a542df433faf size 30786
0479
1#include "llama-model-saver.h"2 3#include "ggml.h"4#include "gguf.h"5 6#include "llama-arch.h"7#include "llama.h"8#include "llama-hparams.h"9#include "llama-model.h"10#include "llama-vocab.h"11 12#include <cstdint>13#include <string>14 15bool llama_model_saver_supports_arch(llm_arch arch) {16 switch (arch) {17 case LLM_ARCH_QWEN3NEXT:18 case LLM_ARCH_QWEN35:19 case LLM_ARCH_QWEN35MOE:20 case LLM_ARCH_PLAMO3:21 case LLM_ARCH_GEMMA3:22 case LLM_ARCH_GEMMA3N:23 case LLM_ARCH_COHERE2:24 case LLM_ARCH_OLMO2:25 case LLM_ARCH_BITNET:26 case LLM_ARCH_T5:27 case LLM_ARCH_EXAONE_MOE:28 case LLM_ARCH_AFMOE:29 case LLM_ARCH_APERTUS:30 case LLM_ARCH_MIMO2:31 case LLM_ARCH_STEP35:32 return false;33 default:34 return true;35 }36}37 38llama_model_saver::llama_model_saver(const struct llama_model * model) :39 gguf_ctx(gguf_init_empty()), gguf_ctx_owned(true), model(model), llm_kv(model->arch) {40 GGML_ASSERT(llama_model_saver_supports_arch(model->arch));41}42 43llama_model_saver::llama_model_saver(enum llm_arch arch, struct gguf_context * gguf_ctx) :44 gguf_ctx(gguf_ctx == nullptr ? gguf_init_empty() : gguf_ctx), gguf_ctx_owned(gguf_ctx == nullptr), model(nullptr), llm_kv(arch) {}45 46llama_model_saver::~llama_model_saver() {47 if (gguf_ctx_owned) {48 gguf_free(gguf_ctx);49 }50}51 52void llama_model_saver::add_kv(const enum llm_kv key, const uint32_t value) {53 gguf_set_val_u32(gguf_ctx, llm_kv(key).c_str(), value);54}55 56void llama_model_saver::add_kv(const enum llm_kv key, const int32_t value) {57 gguf_set_val_i32(gguf_ctx, llm_kv(key).c_str(), value);58}59 60void llama_model_saver::add_kv(const enum llm_kv key, const float value) {61 gguf_set_val_f32(gguf_ctx, llm_kv(key).c_str(), value);62}63 64void llama_model_saver::add_kv(const enum llm_kv key, const bool value) {65 gguf_set_val_bool(gguf_ctx, llm_kv(key).c_str(), value);66}67 68void llama_model_saver::add_kv(const enum llm_kv key, const char * value) {69 gguf_set_val_str(gguf_ctx, llm_kv(key).c_str(), value);70}71 72[[noreturn]]73void llama_model_saver::add_kv(const enum llm_kv key, const char value) {74 GGML_UNUSED(key);75 GGML_UNUSED(value);76 GGML_ABORT("fatal error"); // this should never be called, only needed to make the template below compile77}78 79template <typename Container>80void llama_model_saver::add_kv(const enum llm_kv key, const Container & value, const bool per_layer) {81 GGML_ASSERT(model != nullptr || !per_layer);82 const size_t n_values = per_layer ? size_t(model->hparams.n_layer) : value.size();83 GGML_ASSERT(n_values <= value.size());84 85 if (n_values == 0) {86 return;87 }88 89 if (per_layer) {90 bool all_values_the_same = true;91 for (size_t i = 1; i < n_values; ++i) {92 if (value[i] != value[0]) {93 all_values_the_same = false;94 break;95 }96 }97 if (all_values_the_same) {98 add_kv(key, value[0]);99 return;100 }101 }102 103 if (std::is_same<typename Container::value_type, uint8_t>::value) {104 gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_UINT8, value.data(), n_values);105 } else if (std::is_same<typename Container::value_type, int8_t>::value) {106 gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_INT8, value.data(), n_values);107 } else if (std::is_same<typename Container::value_type, uint32_t>::value) {108 gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_UINT32, value.data(), n_values);109 } else if (std::is_same<typename Container::value_type, int32_t>::value) {110 gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_INT32, value.data(), n_values);111 } else if (std::is_same<typename Container::value_type, float>::value) {112 gguf_set_arr_data(gguf_ctx, llm_kv(key).c_str(), GGUF_TYPE_FLOAT32, value.data(), n_values);113 } else if (std::is_same<Container, std::string>::value) {114 gguf_set_val_str(gguf_ctx, llm_kv(key).c_str(), reinterpret_cast<const char *>(value.data()));115 } else {116 GGML_ABORT("fatal error");117 }118}119// instantiate for external usage:120template void llama_model_saver::add_kv<std::vector<uint32_t>>(const enum llm_kv, const std::vector<uint32_t> &, const bool);121 122void llama_model_saver::add_kv(const enum llm_kv key, const std::vector<std::string> & value) {123 std::vector<const char *> tmp(value.size());124 for (size_t i = 0; i < value.size(); ++i) {125 tmp[i] = value[i].c_str();126 }127 gguf_set_arr_str(gguf_ctx, llm_kv(key).c_str(), tmp.data(), tmp.size());128}129 130void llama_model_saver::add_tensor(const struct ggml_tensor * tensor) {131 if (!tensor) {132 return;133 }134 if (gguf_find_tensor(gguf_ctx, tensor->name) >= 0) {135 const std::string tensor_name = tensor->name;136 GGML_ASSERT(137 tensor_name == "rope_freqs.weight" || tensor_name == "rope_factors_long.weight" ||138 tensor_name == "rope_factors_short.weight"); // FIXME139 return;140 }141 gguf_add_tensor(gguf_ctx, tensor);142}143 144void llama_model_saver::add_kv_from_model() {145 const llama_hparams & hparams = model->hparams;146 const llama_vocab & vocab = model->vocab;147 148 const int32_t n_vocab = vocab.n_tokens();149 std::vector<std::string> tokens(n_vocab);150 std::vector<float> scores(n_vocab);151 std::vector<int32_t> token_types(n_vocab);152 153 if (vocab.get_type() != LLAMA_VOCAB_TYPE_NONE) {154 for (int32_t id = 0; id < n_vocab; ++id) {155 const llama_vocab::token_data & token_data = vocab.get_token_data(id);156 157 tokens[id] = token_data.text;158 scores[id] = token_data.score;159 160 // FIXME should this be treated as flags?161 switch(token_data.attr) {162 case LLAMA_TOKEN_ATTR_UNKNOWN: token_types[id] = LLAMA_TOKEN_TYPE_UNKNOWN; break;163 case LLAMA_TOKEN_ATTR_UNUSED: token_types[id] = LLAMA_TOKEN_TYPE_UNUSED; break;164 case LLAMA_TOKEN_ATTR_NORMAL: token_types[id] = LLAMA_TOKEN_TYPE_NORMAL; break;165 case LLAMA_TOKEN_ATTR_CONTROL: token_types[id] = LLAMA_TOKEN_TYPE_CONTROL; break;166 case LLAMA_TOKEN_ATTR_USER_DEFINED: token_types[id] = LLAMA_TOKEN_TYPE_USER_DEFINED; break;167 case LLAMA_TOKEN_ATTR_BYTE: token_types[id] = LLAMA_TOKEN_TYPE_BYTE; break;168 // case LLAMA_TOKEN_ATTR_NORMALIZED: ???169 // case LLAMA_TOKEN_ATTR_LSTRIP: ???170 // case LLAMA_TOKEN_ATTR_RSTRIP: ???171 case LLAMA_TOKEN_ATTR_UNDEFINED:172 default: token_types[id] = LLAMA_TOKEN_TYPE_UNDEFINED; break;173 }174 }175 }176 177 // add_kv(LLM_KV_GENERAL_TYPE, ???);178 add_kv(LLM_KV_GENERAL_ARCHITECTURE, model->arch_name());179 // add_kv(LLM_KV_GENERAL_QUANTIZATION_VERSION, ???);180 // add_kv(LLM_KV_GENERAL_ALIGNMENT, ???);181 // add_kv(LLM_KV_GENERAL_FILE_TYPE, ???);182 // add_kv(LLM_KV_GENERAL_SAMPLING_SEQUENCE, ???);183 // add_kv(LLM_KV_GENERAL_SAMPLING_TOP_K, ???);184 // add_kv(LLM_KV_GENERAL_SAMPLING_TOP_P, ???);185 // add_kv(LLM_KV_GENERAL_SAMPLING_MIN_P, ???);186 // add_kv(LLM_KV_GENERAL_SAMPLING_XTC_PROBABILITY, ???);187 // add_kv(LLM_KV_GENERAL_SAMPLING_XTC_THRESHOLD, ???);188 // add_kv(LLM_KV_GENERAL_SAMPLING_TEMP, ???);189 // add_kv(LLM_KV_GENERAL_SAMPLING_PENALTY_LAST_N, ???);190 // add_kv(LLM_KV_GENERAL_SAMPLING_PENALTY_REPEAT, ???);191 // add_kv(LLM_KV_GENERAL_SAMPLING_MIROSTAT, ???);192 // add_kv(LLM_KV_GENERAL_SAMPLING_MIROSTAT_TAU, ???);193 // add_kv(LLM_KV_GENERAL_SAMPLING_MIROSTAT_ETA, ???);194 add_kv(LLM_KV_GENERAL_NAME, model->name);195 // add_kv(LLM_KV_GENERAL_AUTHOR, ???);196 // add_kv(LLM_KV_GENERAL_VERSION, ???);197 // add_kv(LLM_KV_GENERAL_URL, ???);198 // add_kv(LLM_KV_GENERAL_DESCRIPTION, ???);199 // add_kv(LLM_KV_GENERAL_LICENSE, ???);200 // add_kv(LLM_KV_GENERAL_SOURCE_URL, ???);201 // add_kv(LLM_KV_GENERAL_SOURCE_HF_REPO, ???);202 203 add_kv(LLM_KV_VOCAB_SIZE, vocab.n_tokens());204 add_kv(LLM_KV_CONTEXT_LENGTH, hparams.n_ctx_train);205 add_kv(LLM_KV_EMBEDDING_LENGTH, hparams.n_embd);206 if (hparams.n_embd_out_impl > 0) {207 add_kv(LLM_KV_EMBEDDING_LENGTH_OUT, hparams.n_embd_out_impl);208 }209 add_kv(LLM_KV_BLOCK_COUNT, hparams.n_layer);210 add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead);211 add_kv(LLM_KV_FEED_FORWARD_LENGTH, hparams.n_ff_arr, true);212 add_kv(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);213 add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp);214 add_kv(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_chexp);215 add_kv(LLM_KV_SWIGLU_CLAMP_EXP, hparams.swiglu_clamp_exp);216 add_kv(LLM_KV_SWIGLU_CLAMP_SHEXP, hparams.swiglu_clamp_shexp);217 add_kv(LLM_KV_USE_PARALLEL_RESIDUAL, hparams.use_par_res);218 // add_kv(LLM_KV_TENSOR_DATA_LAYOUT, ???);219 add_kv(LLM_KV_EXPERT_COUNT, hparams.n_expert);220 add_kv(LLM_KV_EXPERT_USED_COUNT, hparams.n_expert_used);221 add_kv(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared);222 add_kv(LLM_KV_EXPERT_GROUP_COUNT, hparams.n_expert_groups);223 add_kv(LLM_KV_EXPERT_GROUP_USED_COUNT, hparams.n_group_used);224 add_kv(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale);225 add_kv(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm);226 add_kv(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func);227 add_kv(LLM_KV_EXPERT_GROUP_SCALE, hparams.expert_group_scale);228 add_kv(LLM_KV_EXPERTS_PER_GROUP, hparams.n_group_experts);229 add_kv(LLM_KV_MOE_EVERY_N_LAYERS, hparams.moe_every_n_layers);230 add_kv(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.nextn_predict_layers);231 add_kv(LLM_KV_NUM_DEEPSTACK_LAYERS, hparams.n_deepstack_layers);232 add_kv(LLM_KV_POOLING_TYPE, uint32_t(hparams.pooling_type));233 add_kv(LLM_KV_LOGIT_SCALE, hparams.f_logit_scale);234 add_kv(LLM_KV_DECODER_START_TOKEN_ID, hparams.dec_start_token_id);235 add_kv(LLM_KV_DECODER_BLOCK_COUNT, hparams.dec_n_layer);236 add_kv(LLM_KV_ATTN_LOGIT_SOFTCAPPING, hparams.f_attn_logit_softcapping);237 add_kv(LLM_KV_ROUTER_LOGIT_SOFTCAPPING, hparams.f_router_logit_softcapping);238 add_kv(LLM_KV_FINAL_LOGIT_SOFTCAPPING, hparams.f_final_logit_softcapping);239 add_kv(LLM_KV_SWIN_NORM, hparams.swin_norm);240 add_kv(LLM_KV_RESCALE_EVERY_N_LAYERS, hparams.rescale_every_n_layers);241 add_kv(LLM_KV_TIME_MIX_EXTRA_DIM, hparams.time_mix_extra_dim);242 add_kv(LLM_KV_TIME_DECAY_EXTRA_DIM, hparams.time_decay_extra_dim);243 add_kv(LLM_KV_RESIDUAL_SCALE, hparams.f_residual_scale);244 add_kv(LLM_KV_EMBEDDING_SCALE, hparams.f_embedding_scale);245 add_kv(LLM_KV_TOKEN_SHIFT_COUNT, hparams.token_shift_count);246 add_kv(LLM_KV_INTERLEAVE_MOE_LAYER_STEP, hparams.n_moe_layer_step);247 // add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, ???);248 249 add_kv(LLM_KV_ATTENTION_HEAD_COUNT, hparams.n_head_arr, true);250 add_kv(LLM_KV_ATTENTION_HEAD_COUNT_KV, hparams.n_head_kv_arr, true);251 add_kv(LLM_KV_ATTENTION_MAX_ALIBI_BIAS, hparams.f_max_alibi_bias);252 add_kv(LLM_KV_ATTENTION_CLAMP_KQV, hparams.f_clamp_kqv);253 add_kv(LLM_KV_ATTENTION_KEY_LENGTH, hparams.n_embd_head_k_full);254 add_kv(LLM_KV_ATTENTION_VALUE_LENGTH, hparams.n_embd_head_v_full);255 add_kv(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);256 add_kv(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);257 add_kv(LLM_KV_ATTENTION_GROUPNORM_EPS, hparams.f_norm_group_eps);258 add_kv(LLM_KV_ATTENTION_GROUPNORM_GROUPS, hparams.n_norm_groups);259 add_kv(LLM_KV_ATTENTION_CAUSAL, hparams.causal_attn);260 add_kv(LLM_KV_ATTENTION_Q_LORA_RANK, hparams.n_lora_q);261 add_kv(LLM_KV_ATTENTION_KV_LORA_RANK, hparams.n_lora_kv);262 add_kv(LLM_KV_ATTENTION_DECAY_LORA_RANK, hparams.n_lora_decay);263 add_kv(LLM_KV_ATTENTION_ICLR_LORA_RANK, hparams.n_lora_iclr);264 add_kv(LLM_KV_ATTENTION_VALUE_RESIDUAL_MIX_LORA_RANK, hparams.n_lora_value_res_mix);265 add_kv(LLM_KV_ATTENTION_GATE_LORA_RANK, hparams.n_lora_gate);266 add_kv(LLM_KV_ATTENTION_RELATIVE_BUCKETS_COUNT, hparams.n_rel_attn_bkts);267 add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);268 // add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, ???);269 add_kv(LLM_KV_ATTENTION_SCALE, hparams.f_attention_scale);270 add_kv(LLM_KV_ATTENTION_OUTPUT_SCALE, hparams.f_attn_out_scale);271 add_kv(LLM_KV_ATTENTION_TEMPERATURE_LENGTH, hparams.attn_temp_length);272 add_kv(LLM_KV_ATTENTION_TEMPERATURE_SCALE, hparams.f_attn_temp_scale);273 add_kv(LLM_KV_ATTENTION_KEY_LENGTH_MLA, hparams.n_embd_head_k_mla_impl);274 add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_MLA, hparams.n_embd_head_v_mla_impl);275 add_kv(LLM_KV_ATTENTION_KEY_LENGTH_SWA, hparams.n_embd_head_k_swa);276 add_kv(LLM_KV_ATTENTION_VALUE_LENGTH_SWA, hparams.n_embd_head_v_swa);277 add_kv(LLM_KV_ATTENTION_INDEXER_HEAD_COUNT, hparams.indexer_n_head);278 add_kv(LLM_KV_ATTENTION_INDEXER_KEY_LENGTH, hparams.indexer_head_size);279 add_kv(LLM_KV_ATTENTION_INDEXER_TOP_K, hparams.indexer_top_k);280 281 const float rope_scaling_factor = hparams.rope_freq_scale_train == 1.0f ? 0.0f : 1.0f/hparams.rope_freq_scale_train;282 283 add_kv(LLM_KV_ROPE_DIMENSION_COUNT, hparams.n_rot_full);284 add_kv(LLM_KV_ROPE_DIMENSION_COUNT_SWA, hparams.n_rot_swa);285 add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS, hparams.rope_sections);286 add_kv(LLM_KV_ROPE_FREQ_BASE, hparams.rope_freq_base_train);287 add_kv(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa);288 // add_kv(LLM_KV_ROPE_SCALE_LINEAR, rope_scaling_factor); // old name289 add_kv(LLM_KV_ROPE_SCALING_TYPE, llama_rope_scaling_type_name(hparams.rope_scaling_type_train));290 add_kv(LLM_KV_ROPE_SCALING_FACTOR, rope_scaling_factor);291 add_kv(LLM_KV_ROPE_SCALING_ATTN_FACTOR, hparams.rope_attn_factor);292 add_kv(LLM_KV_ROPE_SCALING_ORIG_CTX_LEN, hparams.n_ctx_orig_yarn);293 add_kv(LLM_KV_ROPE_SCALING_FINETUNED, hparams.rope_finetuned);294 add_kv(LLM_KV_ROPE_SCALING_YARN_LOG_MUL, hparams.rope_yarn_log_mul);295 add_kv(LLM_KV_ROPE_SCALING_YARN_EXT_FACTOR, hparams.yarn_ext_factor);296 add_kv(LLM_KV_ROPE_SCALING_YARN_ATTN_FACTOR, hparams.yarn_attn_factor);297 add_kv(LLM_KV_ROPE_SCALING_YARN_BETA_FAST, hparams.yarn_beta_fast);298 add_kv(LLM_KV_ROPE_SCALING_YARN_BETA_SLOW, hparams.yarn_beta_slow);299 300 // TODO: implement split file support301 // add_kv(LLM_KV_SPLIT_NO, ???);302 // add_kv(LLM_KV_SPLIT_COUNT, ???);303 // add_kv(LLM_KV_SPLIT_TENSORS_COUNT, ???);304 305 add_kv(LLM_KV_SSM_INNER_SIZE, hparams.ssm_d_inner);306 add_kv(LLM_KV_SSM_CONV_KERNEL, hparams.ssm_d_conv);307 add_kv(LLM_KV_SSM_STATE_SIZE, hparams.ssm_d_state);308 add_kv(LLM_KV_SSM_TIME_STEP_RANK, hparams.ssm_dt_rank);309 add_kv(LLM_KV_SSM_GROUP_COUNT, hparams.ssm_n_group);310 add_kv(LLM_KV_SSM_DT_B_C_RMS, hparams.ssm_dt_b_c_rms);311 312 add_kv(LLM_KV_KDA_HEAD_DIM, hparams.n_embd_head_kda);313 314 add_kv(LLM_KV_WKV_HEAD_SIZE, hparams.wkv_head_size);315 316 add_kv(LLM_KV_TOKENIZER_MODEL, vocab.get_tokenizer_model());317 add_kv(LLM_KV_TOKENIZER_PRE, vocab.get_tokenizer_pre());318 add_kv(LLM_KV_TOKENIZER_LIST, tokens);319 add_kv(LLM_KV_TOKENIZER_TOKEN_TYPE, token_types);320 add_kv(LLM_KV_TOKENIZER_TOKEN_TYPE_COUNT, vocab.n_token_types());321 add_kv(LLM_KV_TOKENIZER_SCORES, scores);322 add_kv(LLM_KV_TOKENIZER_MERGES, vocab.get_bpe_merges());323 // FIXME llama_token is type i32 but when reading in a GGUF file u32 is expected, not an issue for writing though324 add_kv(LLM_KV_TOKENIZER_BOS_ID, uint32_t(vocab.token_bos()));325 add_kv(LLM_KV_TOKENIZER_EOS_ID, uint32_t(vocab.token_eos()));326 add_kv(LLM_KV_TOKENIZER_EOT_ID, uint32_t(vocab.token_eot()));327 add_kv(LLM_KV_TOKENIZER_EOM_ID, uint32_t(vocab.token_eom()));328 add_kv(LLM_KV_TOKENIZER_UNK_ID, uint32_t(vocab.token_unk()));329 add_kv(LLM_KV_TOKENIZER_SEP_ID, uint32_t(vocab.token_sep()));330 add_kv(LLM_KV_TOKENIZER_PAD_ID, uint32_t(vocab.token_pad()));331 // add_kv(LLM_KV_TOKENIZER_CLS_ID, uint32_t(vocab.token_bos())); // deprecated332 // add_kv(LLM_KV_TOKENIZER_MASK_ID, ???);333 add_kv(LLM_KV_TOKENIZER_ADD_BOS, vocab.get_add_bos());334 add_kv(LLM_KV_TOKENIZER_ADD_EOS, vocab.get_add_eos());335 add_kv(LLM_KV_TOKENIZER_ADD_SEP, vocab.get_add_sep());336 add_kv(LLM_KV_TOKENIZER_ADD_PREFIX, vocab.get_add_space_prefix());337 add_kv(LLM_KV_TOKENIZER_REMOVE_EXTRA_WS, vocab.get_remove_extra_whitespaces());338 add_kv(LLM_KV_TOKENIZER_PRECOMPILED_CHARSMAP, vocab.get_precompiled_charsmap());339 // add_kv(LLM_KV_TOKENIZER_HF_JSON, ???);340 // add_kv(LLM_KV_TOKENIZER_RWKV, ???);341 add_kv(LLM_KV_TOKENIZER_FIM_PRE_ID, uint32_t(vocab.token_fim_pre()));342 add_kv(LLM_KV_TOKENIZER_FIM_SUF_ID, uint32_t(vocab.token_fim_suf()));343 add_kv(LLM_KV_TOKENIZER_FIM_MID_ID, uint32_t(vocab.token_fim_mid()));344 add_kv(LLM_KV_TOKENIZER_FIM_PAD_ID, uint32_t(vocab.token_fim_pad()));345 add_kv(LLM_KV_TOKENIZER_FIM_REP_ID, uint32_t(vocab.token_fim_rep()));346 add_kv(LLM_KV_TOKENIZER_FIM_SEP_ID, uint32_t(vocab.token_fim_sep()));347 348 // TODO: implement LoRA support349 // add_kv(LLM_KV_ADAPTER_TYPE, ???);350 // add_kv(LLM_KV_ADAPTER_LORA_ALPHA, ???);351 // add_kv(LLM_KV_ADAPTER_LORA_TASK_NAME, ???);352 // add_kv(LLM_KV_ADAPTER_LORA_PROMPT_PREFIX, ???);353 // add_kv(LLM_KV_ADAPTER_ALORA_INVOCATION_TOKENS, ???);354 355 add_kv(LLM_KV_POSNET_EMBEDDING_LENGTH, hparams.posnet.n_embd);356 add_kv(LLM_KV_POSNET_BLOCK_COUNT, hparams.posnet.n_layer);357 358 add_kv(LLM_KV_CONVNEXT_EMBEDDING_LENGTH, hparams.convnext.n_embd);359 add_kv(LLM_KV_CONVNEXT_BLOCK_COUNT, hparams.convnext.n_layer);360 361 add_kv(LLM_KV_CLASSIFIER_OUTPUT_LABELS, model->classifier_labels);362 363 add_kv(LLM_KV_SHORTCONV_L_CACHE, hparams.n_shortconv_l_cache);364 365 add_kv(LLM_KV_XIELU_ALPHA_N, hparams.xielu_alpha_n);366 add_kv(LLM_KV_XIELU_ALPHA_P, hparams.xielu_alpha_p);367 add_kv(LLM_KV_XIELU_BETA, hparams.xielu_beta);368 add_kv(LLM_KV_XIELU_EPS, hparams.xielu_eps);369 370 // deprecated371 // add_kv(LLM_KV_TOKENIZER_PREFIX_ID, ???);372 // add_kv(LLM_KV_TOKENIZER_SUFFIX_ID, ???);373 // add_kv(LLM_KV_TOKENIZER_MIDDLE_ID, ???);374 375 add_kv(LLM_KV_DENSE_2_FEAT_IN, hparams.dense_2_feat_in);376 add_kv(LLM_KV_DENSE_2_FEAT_OUT, hparams.dense_2_feat_out);377 add_kv(LLM_KV_DENSE_3_FEAT_IN, hparams.dense_3_feat_in);378 add_kv(LLM_KV_DENSE_3_FEAT_OUT, hparams.dense_3_feat_out);379}380 381void llama_model_saver::add_tensors_from_model() {382 if (model->output != nullptr &&383 std::string(model->output->name) != std::string(model->tok_embd->name)) {384 add_tensor(model->tok_embd); // some models use the same tensor for tok_embd and output385 }386 add_tensor(model->type_embd);387 add_tensor(model->pos_embd);388 add_tensor(model->tok_norm);389 add_tensor(model->tok_norm_b);390 add_tensor(model->output_norm);391 add_tensor(model->output_norm_b);392 add_tensor(model->output);393 add_tensor(model->output_b);394 add_tensor(model->output_norm_enc);395 add_tensor(model->cls);396 add_tensor(model->cls_b);397 add_tensor(model->cls_out);398 add_tensor(model->cls_out_b);399 add_tensor(model->cls_norm);400 401 for (const struct llama_layer & layer : model->layers) {402 for (size_t i = 0; i < sizeof(layer)/sizeof(struct ggml_tensor *); ++i) {403 add_tensor(reinterpret_cast<const struct ggml_tensor * const *>(&layer)[i]);404 }405 }406}407 408void llama_model_saver::save(const std::string & path_model) {409 gguf_write_to_file(gguf_ctx, path_model.c_str(), false);410}411 412void llama_model_saver::save(FILE * file) {413 gguf_write_to_file_ptr(gguf_ctx, file, false);414}415