Brunobkr/llama.cpp_AlgMor24_github
ΩFFFΣLLIa • llama.cpp • AlgMor24 ██████╗ ███████╗███████╗███████╗██╗ ██╗ ██╗ █████╗ ██╔═══██╗██╔════╝██╔════╝██╔════╝██║ ██║ ██║██╔══██╗ ██║ ██║█████╗ █████╗ █████╗ ██║ ██║ ██║███████║ ██║ ██║██╔══╝ ██╔══╝ ██╔══╝ ██║ ██║ ██║██╔══██║ ╚██████╔╝██║ ██║ ███████╗███████╗███████╗██║██║ ██║ ╚═════╝ ╚═╝ ╚═╝ ╚══════╝╚══════╝╚══════╝╚═╝╚═╝ ╚═╝ High-Performance LLM / VLM Inference & Autonomous Agentic Ecosystem… See the full description on the dataset page: https://huggingface.co/datasets/Brunobkr/llama.cpp_AlgMor24_github.
03.1k
1#include "llama-impl.h"2#include "llama-model.h"3#include "llama-model-loader.h"4#include "llama-ext.h"5#include "llama.h"6 7#include <algorithm>8#include <cmath>9#include <cstring>10#include <cinttypes>11#include <fstream>12#include <mutex>13#include <regex>14#include <thread>15#include <unordered_map>16 17// result of parsing --tensor-type option18// (changes to this struct must be reflected in tools/quantize/quantize.cpp)19struct tensor_type_option {20 std::string name;21 ggml_type type = GGML_TYPE_COUNT;22};23 24// tensor categorization - used to avoid repeated string matching in quantization logic.25// this is different from LLM_TN - we want broad categories, not specific tensor names per arch.26enum class tensor_category {27 TOKEN_EMBD,28 ATTENTION_Q,29 ATTENTION_V,30 ATTENTION_K,31 ATTENTION_QKV,32 ATTENTION_KV_B,33 ATTENTION_OUTPUT,34 FFN_UP,35 FFN_GATE,36 FFN_DOWN,37 OUTPUT,38 OTHER39};40 41static void zeros(std::ofstream & file, size_t n) {42 char zero = 0;43 for (size_t i = 0; i < n; ++i) {44 file.write(&zero, 1);45 }46}47 48static std::string remap_layer(const std::string & orig_name, const std::vector<int> & prune, std::map<int, std::string> & mapped, int & next_id) {49 if (prune.empty()) {50 return orig_name;51 }52 53 static const std::regex pattern(R"(blk\.(\d+)\.)");54 if (std::smatch match; std::regex_search(orig_name, match, pattern)) {55 const int blk = std::stoi(match[1]);56 std::string new_name = orig_name;57 58 if (mapped.count(blk)) {59 // Already mapped, do nothing60 } else if (std::find(prune.begin(), prune.end(), blk) != prune.end()) {61 mapped[blk] = "";62 } else if (blk < prune.front()) {63 mapped[blk] = std::to_string(blk);64 next_id = blk + 1;65 } else {66 mapped[blk] = std::to_string(next_id);67 ++next_id;68 }69 70 return mapped[blk].empty() ? mapped[blk] : new_name.replace(match.position(1), match.length(1), mapped[blk]);71 }72 73 return orig_name;74}75 76static std::string remap_imatrix(const std::string & orig_name, const std::map<int, std::string> & mapped) {77 if (mapped.empty()) {78 return orig_name;79 }80 81 static const std::regex pattern(R"(blk\.(\d+)\.)");82 if (std::smatch match; std::regex_search(orig_name, match, pattern)) {83 const std::string blk(match[1]);84 std::string new_name = orig_name;85 86 for (const auto & p : mapped) {87 if (p.second == blk) {88 return new_name.replace(match.position(1), match.length(1), std::to_string(p.first));89 }90 }91 GGML_ABORT("\n%s: imatrix mapping error for %s\n", __func__, orig_name.c_str());92 }93 94 return orig_name;95}96 97//98// helper functions for tensor name matching99//100 101static bool tensor_name_match_token_embd(const char * tensor_name) {102 return std::strcmp(tensor_name, "token_embd.weight") == 0 ||103 std::strcmp(tensor_name, "per_layer_token_embd.weight") == 0;104}105 106static bool tensor_name_match_output_weight(const char * tensor_name) {107 return std::strcmp(tensor_name, "output.weight") == 0;108}109 110//111// tensor categorization for quantization112//113// (this is different from LLM_TN - we want broad categories, not specific tensor names per arch)114//115 116static tensor_category tensor_get_category(const std::string & tensor_name) {117 if (tensor_name_match_output_weight(tensor_name.c_str())) {118 return tensor_category::OUTPUT;119 }120 if (tensor_name_match_token_embd(tensor_name.c_str())) {121 return tensor_category::TOKEN_EMBD;122 }123 if (tensor_name.find("attn_qkv.weight") != std::string::npos) {124 return tensor_category::ATTENTION_QKV;125 }126 if (tensor_name.find("attn_kv_b.weight") != std::string::npos) {127 return tensor_category::ATTENTION_KV_B;128 }129 if (tensor_name.find("attn_v.weight") != std::string::npos) {130 return tensor_category::ATTENTION_V;131 }132 if (tensor_name.find("attn_k.weight") != std::string::npos) {133 return tensor_category::ATTENTION_K;134 }135 if (tensor_name.find("attn_q.weight") != std::string::npos) {136 return tensor_category::ATTENTION_Q;137 }138 if (tensor_name.find("attn_output.weight") != std::string::npos) {139 return tensor_category::ATTENTION_OUTPUT;140 }141 if (tensor_name.find("ffn_up") != std::string::npos) {142 return tensor_category::FFN_UP;143 }144 if (tensor_name.find("ffn_gate") != std::string::npos) {145 return tensor_category::FFN_GATE;146 }147 if (tensor_name.find("ffn_down") != std::string::npos) {148 return tensor_category::FFN_DOWN;149 }150 return tensor_category::OTHER;151}152 153// check if category is for attention-v-like tensors (more sensitive to quantization)154static bool category_is_attn_v(tensor_category cat) {155 return cat == tensor_category::ATTENTION_V ||156 cat == tensor_category::ATTENTION_QKV ||157 cat == tensor_category::ATTENTION_KV_B;158}159 160//161// quantization state162//163 164struct quantize_state_impl {165 const llama_model & model;166 const llama_model_quantize_params * params;167 168 int n_attention_wv = 0;169 int n_ffn_down = 0;170 int n_ffn_gate = 0;171 int n_ffn_up = 0;172 int i_attention_wv = 0;173 int i_ffn_down = 0;174 int i_ffn_gate = 0;175 int i_ffn_up = 0;176 177 int n_fallback = 0;178 179 bool has_imatrix = false;180 181 // used to figure out if a model has tied embeddings (tok_embd shares weights with output)182 bool has_tied_embeddings = true; // assume tied until we see output.weight183 184 // tensor type override patterns (compiled once, used twice)185 std::vector<std::pair<std::regex, ggml_type>> tensor_type_patterns;186 187 quantize_state_impl(const llama_model & model, const llama_model_quantize_params * params):188 model(model), params(params)189 {190 // compile regex patterns once - they are expensive191 if (params->tt_overrides) {192 for (const auto * p = params->tt_overrides; p->pattern != nullptr; p++) {193 tensor_type_patterns.emplace_back(std::regex(p->pattern), p->type);194 }195 }196 }197};198 199// per-tensor metadata, computed in the preliminary loop and used in the main loop200struct tensor_metadata {201 std::string name;202 ggml_type target_type;203 tensor_category category;204 std::string remapped_imatrix_name;205 bool allows_quantization;206 bool requires_imatrix;207};208 209//210// dequantization211//212 213static void llama_tensor_dequantize_impl(214 ggml_tensor * tensor, std::vector<no_init<float>> & output, std::vector<std::thread> & workers,215 const size_t nelements, const int nthread216) {217 if (output.size() < nelements) {218 output.resize(nelements);219 }220 float * f32_output = (float *) output.data();221 222 const ggml_type_traits * qtype = ggml_get_type_traits(tensor->type);223 if (ggml_is_quantized(tensor->type)) {224 if (qtype->to_float == NULL) {225 throw std::runtime_error(format("type %s unsupported for integer quantization: no dequantization available", ggml_type_name(tensor->type)));226 }227 } else if (tensor->type != GGML_TYPE_F16 &&228 tensor->type != GGML_TYPE_BF16) {229 throw std::runtime_error(format("cannot dequantize/convert tensor type %s", ggml_type_name(tensor->type)));230 }231 232 if (nthread < 2) {233 if (tensor->type == GGML_TYPE_F16) {234 ggml_fp16_to_fp32_row((ggml_fp16_t *)tensor->data, f32_output, nelements);235 } else if (tensor->type == GGML_TYPE_BF16) {236 ggml_bf16_to_fp32_row((ggml_bf16_t *)tensor->data, f32_output, nelements);237 } else if (ggml_is_quantized(tensor->type)) {238 qtype->to_float(tensor->data, f32_output, nelements);239 } else {240 GGML_ABORT("fatal error"); // unreachable241 }242 return;243 }244 245 size_t block_size;246 if (tensor->type == GGML_TYPE_F16 ||247 tensor->type == GGML_TYPE_BF16) {248 block_size = 1;249 } else {250 block_size = (size_t)ggml_blck_size(tensor->type);251 }252 253 size_t block_size_bytes = ggml_type_size(tensor->type);254 255 GGML_ASSERT(nelements % block_size == 0);256 size_t nblocks = nelements / block_size;257 size_t blocks_per_thread = nblocks / nthread;258 size_t spare_blocks = nblocks - (blocks_per_thread * nthread); // if blocks aren't divisible by thread count259 260 size_t in_buff_offs = 0;261 size_t out_buff_offs = 0;262 263 for (int tnum = 0; tnum < nthread; tnum++) {264 size_t thr_blocks = blocks_per_thread + (tnum == nthread - 1 ? spare_blocks : 0); // num blocks for this thread265 size_t thr_elems = thr_blocks * block_size; // number of elements for this thread266 size_t thr_block_bytes = thr_blocks * block_size_bytes; // number of input bytes for this thread267 268 auto compute = [qtype] (ggml_type typ, uint8_t * inbuf, float * outbuf, int nels) {269 if (typ == GGML_TYPE_F16) {270 ggml_fp16_to_fp32_row((ggml_fp16_t *)inbuf, outbuf, nels);271 } else if (typ == GGML_TYPE_BF16) {272 ggml_bf16_to_fp32_row((ggml_bf16_t *)inbuf, outbuf, nels);273 } else {274 qtype->to_float(inbuf, outbuf, nels);275 }276 };277 workers.emplace_back(compute, tensor->type, (uint8_t *) tensor->data + in_buff_offs, f32_output + out_buff_offs, thr_elems);278 in_buff_offs += thr_block_bytes;279 out_buff_offs += thr_elems;280 }281 for (auto & w : workers) { w.join(); }282 workers.clear();283}284 285//286// do we allow this tensor to be quantized?287//288 289static bool tensor_allows_quantization(const llama_model_quantize_params * params, llm_arch arch, const ggml_tensor * tensor) {290 // trivial checks first -- no string ops needed291 if (params->only_copy) return false;292 293 // quantize only 2D and 3D tensors (experts)294 if (ggml_n_dims(tensor) < 2) return false;295 296 const std::string name = ggml_get_name(tensor);297 298 // This used to be a regex, but <regex> has an extreme cost to compile times.299 bool quantize = name.rfind("weight") == name.size() - 6; // ends with 'weight'?300 301 // do not quantize norm tensors302 quantize &= name.find("_norm.weight") == std::string::npos;303 304 quantize &= params->quantize_output_tensor || name != "output.weight";305 306 // do not quantize expert gating tensors307 // NOTE: can't use LLM_TN here because the layer number is not known308 quantize &= name.find("ffn_gate_inp.weight") == std::string::npos;309 310 // do not quantize the i32 token-id -> expert-id routing table (DeepSeek-V4)311 quantize &= name.find("ffn_gate_tid2eid.weight") == std::string::npos;312 313 // these are very small (e.g. 4x4)314 quantize &= name.find("altup") == std::string::npos;315 quantize &= name.find("laurel") == std::string::npos;316 317 // these are not too big so keep them as it is318 quantize &= name.find("per_layer_model_proj") == std::string::npos;319 320 // do not quantize positional embeddings and token types (BERT)321 quantize &= name != LLM_TN(arch)(LLM_TENSOR_POS_EMBD, "weight");322 quantize &= name != LLM_TN(arch)(LLM_TENSOR_TOKEN_TYPES, "weight");323 324 // do not quantize Mamba/Kimi's small conv1d weights325 // NOTE: can't use LLM_TN here because the layer number is not known326 quantize &= name.find("ssm_conv1d") == std::string::npos;327 quantize &= name.find("shortconv.conv.weight") == std::string::npos;328 329 // do not quantize MiniMax's indexer projection weights, they are tiny330 quantize &= name.find("indexer.k_proj.weight") == std::string::npos;331 quantize &= name.find("indexer.q_proj.weight") == std::string::npos;332 333 // do not quantize RWKV's small yet 2D weights334 quantize &= name.find("time_mix_first.weight") == std::string::npos;335 quantize &= name.find("time_mix_w0.weight") == std::string::npos;336 quantize &= name.find("time_mix_w1.weight") == std::string::npos;337 quantize &= name.find("time_mix_w2.weight") == std::string::npos;338 quantize &= name.find("time_mix_v0.weight") == std::string::npos;339 quantize &= name.find("time_mix_v1.weight") == std::string::npos;340 quantize &= name.find("time_mix_v2.weight") == std::string::npos;341 quantize &= name.find("time_mix_a0.weight") == std::string::npos;342 quantize &= name.find("time_mix_a1.weight") == std::string::npos;343 quantize &= name.find("time_mix_a2.weight") == std::string::npos;344 quantize &= name.find("time_mix_g1.weight") == std::string::npos;345 quantize &= name.find("time_mix_g2.weight") == std::string::npos;346 quantize &= name.find("time_mix_decay_w1.weight") == std::string::npos;347 quantize &= name.find("time_mix_decay_w2.weight") == std::string::npos;348 quantize &= name.find("time_mix_lerp_fused.weight") == std::string::npos;349 350 // do not quantize relative position bias (T5)351 quantize &= name.find("attn_rel_b.weight") == std::string::npos;352 353 // do not quantize specific multimodal tensors354 quantize &= name.find(".position_embd") == std::string::npos;355 quantize &= name.find("sam.pos_embd") == std::string::npos;356 quantize &= name.find("sam.neck.") == std::string::npos;357 quantize &= name.find("sam.net_") == std::string::npos;358 quantize &= name.find(".rel_pos") == std::string::npos;359 quantize &= name.find(".patch_embd") == std::string::npos;360 quantize &= name.find(".patch_merger") == std::string::npos;361 362 // audio codebook363 quantize &= name.find("a.rvq.codebook") == std::string::npos;364 quantize &= name.find("mm.a.code_embd") == std::string::npos;365 366 return quantize;367}368 369//370// tensor type selection371//372 373// incompatible tensor shapes are handled here - fallback to a compatible type374static ggml_type tensor_type_fallback(quantize_state_impl & qs, const ggml_tensor * t, const ggml_type target_type) {375 ggml_type return_type = target_type;376 377 const int64_t ncols = t->ne[0];378 const int64_t qk_k = ggml_blck_size(target_type);379 380 if (ncols % qk_k != 0) { // this tensor's shape is incompatible with this quant381 LLAMA_LOG_WARN("warning: %-36s - ncols %6" PRId64 " not divisible by %3" PRId64 " (required for type %7s) ",382 t->name, ncols, qk_k, ggml_type_name(target_type));383 ++qs.n_fallback;384 385 switch (target_type) {386 // types on the left: block size 256387 case GGML_TYPE_IQ1_S:388 case GGML_TYPE_IQ1_M:389 case GGML_TYPE_IQ2_XXS:390 case GGML_TYPE_IQ2_XS:391 case GGML_TYPE_IQ2_S:392 case GGML_TYPE_IQ3_XXS:393 case GGML_TYPE_IQ3_S: // types on the right: block size 32394 case GGML_TYPE_IQ4_XS: return_type = GGML_TYPE_IQ4_NL; break;395 case GGML_TYPE_Q2_0:396 case GGML_TYPE_Q2_K:397 case GGML_TYPE_Q3_K:398 case GGML_TYPE_TQ1_0:399 case GGML_TYPE_TQ2_0: return_type = GGML_TYPE_Q4_0; break;400 case GGML_TYPE_Q4_K: return_type = GGML_TYPE_Q5_0; break;401 case GGML_TYPE_Q5_K: return_type = GGML_TYPE_Q5_1; break;402 case GGML_TYPE_Q6_K: return_type = GGML_TYPE_Q8_0; break;403 default:404 throw std::runtime_error(format("no tensor type fallback is defined for type %s",405 ggml_type_name(target_type)));406 }407 if (ncols % ggml_blck_size(return_type) != 0) {408 //409 // the fallback return type is still not compatible for this tensor!410 //411 // most likely, this tensor's first dimension is not divisible by 32.412 // this is very rare. we can either abort the quantization, or413 // fallback to F16 / F32.414 //415 LLAMA_LOG_WARN("(WARNING: must use F16 due to unusual shape) ");416 return_type = GGML_TYPE_F16;417 }418 LLAMA_LOG_WARN("-> falling back to %7s\n", ggml_type_name(return_type));419 }420 return return_type;421}422 423// internal standard logic for selecting the target tensor type based on tensor category, ftype, and model arch424static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype, tensor_category category) {425 const std::string name = ggml_get_name(tensor);426 427 // TODO: avoid hardcoded tensor names - use the TN_* constants428 const llm_arch arch = qs.model.arch;429 430 auto use_more_bits = [](int i_layer, int n_layers) -> bool {431 return i_layer < n_layers/8 || i_layer >= 7*n_layers/8 || (i_layer - n_layers/8)%3 == 2;432 };433 const int n_expert = std::max(1, (int)qs.model.hparams.n_expert);434 auto layer_info = [n_expert] (int i_layer, int n_layer, const char * name) {435 if (n_expert > 1) {436 // Believe it or not, "experts" in the FFN of Mixtral-8x7B are not consecutive, but occasionally randomly437 // sprinkled in the model. Hence, simply dividing i_ffn_down by n_expert does not work438 // for getting the current layer as I initially thought, and we need to resort to parsing the439 // tensor name.440 if (sscanf(name, "blk.%d.", &i_layer) != 1) {441 throw std::runtime_error(format("Failed to determine layer for tensor %s", name));442 }443 if (i_layer < 0 || i_layer >= n_layer) {444 throw std::runtime_error(format("Bad layer %d for tensor %s. Must be in [0, %d)", i_layer, name, n_layer));445 }446 }447 return std::make_pair(i_layer, n_layer);448 };449 450 // for arches that share the same tensor between the token embeddings and the output, we quantize the token embeddings451 // with the quantization of the output tensor452 if (category == tensor_category::OUTPUT || (qs.has_tied_embeddings && category == tensor_category::TOKEN_EMBD)) {453 if (qs.params->output_tensor_type < GGML_TYPE_COUNT) {454 new_type = qs.params->output_tensor_type;455 } else {456 const int64_t nx = tensor->ne[0];457 const int64_t qk_k = ggml_blck_size(new_type);458 459 if (ftype == LLAMA_FTYPE_MOSTLY_MXFP4_MOE) {460 new_type = GGML_TYPE_Q8_0;461 }462 else if (arch == LLM_ARCH_FALCON || nx % qk_k != 0) {463 new_type = GGML_TYPE_Q8_0;464 }465 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS ||466 ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ||467 ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {468 new_type = GGML_TYPE_Q5_K;469 }470 else if (new_type != GGML_TYPE_Q8_0) {471 new_type = GGML_TYPE_Q6_K;472 }473 }474 } else if (ftype == LLAMA_FTYPE_MOSTLY_MXFP4_MOE) {475 // MoE tensors -> MXFP4476 // other tensors -> Q8_0477 if (tensor->ne[2] > 1) {478 new_type = GGML_TYPE_MXFP4;479 } else {480 new_type = GGML_TYPE_Q8_0;481 }482 } else if (category == tensor_category::TOKEN_EMBD) {483 if (qs.params->token_embedding_type < GGML_TYPE_COUNT) {484 new_type = qs.params->token_embedding_type;485 } else {486 if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS ||487 ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {488 new_type = GGML_TYPE_Q2_K;489 }490 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) {491 new_type = GGML_TYPE_IQ3_S;492 }493 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {494 new_type = GGML_TYPE_IQ3_S;495 }496 else if (ftype == LLAMA_FTYPE_MOSTLY_TQ1_0 || ftype == LLAMA_FTYPE_MOSTLY_TQ2_0 || ftype == LLAMA_FTYPE_MOSTLY_Q2_0) {497 new_type = GGML_TYPE_Q4_K;498 }499 }500 } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S ||501 ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) {502 if (category_is_attn_v(category)) {503 if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K;504 else new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K;505 ++qs.i_attention_wv;506 }507 else if (qs.model.hparams.n_expert == 8 && category == tensor_category::ATTENTION_K) {508 new_type = GGML_TYPE_Q4_K;509 }510 else if (category == tensor_category::FFN_DOWN) {511 if (qs.i_ffn_down < qs.n_ffn_down/8) {512 new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K;513 }514 ++qs.i_ffn_down;515 }516 else if (category == tensor_category::ATTENTION_OUTPUT) {517 if (qs.model.hparams.n_expert == 8) {518 new_type = GGML_TYPE_Q5_K;519 } else {520 if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) new_type = GGML_TYPE_IQ2_XXS;521 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) new_type = GGML_TYPE_IQ3_S;522 }523 }524 } else if (category_is_attn_v(category)) {525 if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) {526 new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;527 }528 else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S && qs.model.hparams.n_gqa() >= 4) {529 new_type = GGML_TYPE_Q4_K;530 }531 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {532 new_type = qs.model.hparams.n_gqa() >= 4 ? GGML_TYPE_Q4_K : !qs.has_imatrix ? GGML_TYPE_IQ3_S : GGML_TYPE_IQ3_XXS;533 }534 else if ((ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S) && qs.model.hparams.n_gqa() >= 4) {535 new_type = GGML_TYPE_Q4_K;536 }537 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) {538 new_type = GGML_TYPE_Q4_K;539 }540 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {541 new_type = qs.i_attention_wv < 2 ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;542 }543 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;544 else if ((ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) && qs.model.hparams.n_gqa() >= 4) {545 new_type = GGML_TYPE_Q5_K;546 }547 else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) &&548 use_more_bits(qs.i_attention_wv, qs.n_attention_wv)) new_type = GGML_TYPE_Q6_K;549 else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && qs.i_attention_wv < 4) new_type = GGML_TYPE_Q5_K;550 if (qs.model.type == LLM_TYPE_70B) {551 // In the 70B model we have 8 heads sharing the same attn_v weights. As a result, the attn_v.weight tensor is552 // 8x smaller compared to attn_q.weight. Hence, we can get a nice boost in quantization accuracy with553 // nearly negligible increase in model size by quantizing this tensor with more bits:554 if (new_type == GGML_TYPE_Q3_K || new_type == GGML_TYPE_Q4_K) new_type = GGML_TYPE_Q5_K;555 }556 if (qs.model.hparams.n_expert == 8) {557 // for the 8-expert model, bumping this to Q8_0 trades just ~128MB558 // TODO: explore better strategies559 new_type = GGML_TYPE_Q8_0;560 }561 ++qs.i_attention_wv;562 } else if (category == tensor_category::ATTENTION_K) {563 if (qs.model.hparams.n_expert == 8) {564 // for the 8-expert model, bumping this to Q8_0 trades just ~128MB565 // TODO: explore better strategies566 new_type = GGML_TYPE_Q8_0;567 }568 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS) {569 new_type = GGML_TYPE_IQ3_XXS;570 }571 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {572 new_type = GGML_TYPE_IQ2_S;573 }574 } else if (category == tensor_category::ATTENTION_Q) {575 if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS) {576 new_type = GGML_TYPE_IQ3_XXS;577 }578 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) {579 new_type = GGML_TYPE_IQ2_S;580 }581 } else if (category == tensor_category::FFN_DOWN) {582 auto info = layer_info(qs.i_ffn_down, qs.n_ffn_down, name.c_str());583 int i_layer = info.first, n_layer = info.second;584 if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q3_K;585 else if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S) {586 if (i_layer < n_layer/8) new_type = GGML_TYPE_Q4_K;587 }588 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS && !qs.has_imatrix) {589 new_type = i_layer < n_layer/8 ? GGML_TYPE_Q4_K : GGML_TYPE_Q3_K;590 }591 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M) {592 new_type = i_layer < n_layer/16 ? GGML_TYPE_Q5_K593 : arch != LLM_ARCH_FALCON || use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q4_K594 : GGML_TYPE_Q3_K;595 }596 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M && (i_layer < n_layer/8 ||597 (qs.model.hparams.n_expert == 8 && use_more_bits(i_layer, n_layer)))) {598 new_type = GGML_TYPE_Q4_K;599 }600 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) {601 new_type = arch == LLM_ARCH_FALCON ? GGML_TYPE_Q4_K : GGML_TYPE_Q5_K;602 }603 else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) {604 if (arch == LLM_ARCH_FALCON) {605 new_type = i_layer < n_layer/16 ? GGML_TYPE_Q6_K :606 use_more_bits(i_layer, n_layer) ? GGML_TYPE_Q5_K : GGML_TYPE_Q4_K;607 } else {608 if (use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K;609 }610 }611 else if (i_layer < n_layer/8 && (ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) && !qs.has_imatrix) {612 new_type = GGML_TYPE_Q5_K;613 }614 else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M && use_more_bits(i_layer, n_layer)) new_type = GGML_TYPE_Q6_K;615 else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S && arch != LLM_ARCH_FALCON && i_layer < n_layer/8) {616 new_type = GGML_TYPE_Q5_K;617 }618 else if ((ftype == LLAMA_FTYPE_MOSTLY_Q4_0 || ftype == LLAMA_FTYPE_MOSTLY_Q5_0)619 && qs.has_imatrix && i_layer < n_layer/8) {620 // Guard against craziness in the first few ffn_down layers that can happen even with imatrix for Q4_0/Q5_0.621 // We only do it when an imatrix is provided because a) we want to make sure that one can always get the622 // same quantization as before imatrix stuff, and b) Q4_1/Q5_1 do go crazy on ffn_down without an imatrix.623 new_type = ftype == LLAMA_FTYPE_MOSTLY_Q4_0 ? GGML_TYPE_Q4_1 : GGML_TYPE_Q5_1;624 }625 ++qs.i_ffn_down;626 } else if (category == tensor_category::ATTENTION_OUTPUT) {627 if (arch != LLM_ARCH_FALCON) {628 if (qs.model.hparams.n_expert == 8) {629 if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS ||630 ftype == LLAMA_FTYPE_MOSTLY_Q3_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_NL ||631 ftype == LLAMA_FTYPE_MOSTLY_Q4_K_S || ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M || ftype == LLAMA_FTYPE_MOSTLY_IQ3_S ||632 ftype == LLAMA_FTYPE_MOSTLY_IQ3_M || ftype == LLAMA_FTYPE_MOSTLY_IQ4_XS) {633 new_type = GGML_TYPE_Q5_K;634 }635 } else {636 if (ftype == LLAMA_FTYPE_MOSTLY_Q2_K ) new_type = GGML_TYPE_Q3_K;637 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS) new_type = GGML_TYPE_IQ3_S;638 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M ) new_type = GGML_TYPE_Q4_K;639 else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L ) new_type = GGML_TYPE_Q5_K;640 else if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_M ) new_type = GGML_TYPE_Q4_K;641 }642 } else {643 if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q4_K;644 }645 }646 else if (category == tensor_category::ATTENTION_QKV) {647 if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L || ftype == LLAMA_FTYPE_MOSTLY_IQ3_M) {648 new_type = GGML_TYPE_Q4_K;649 }650 else if (ftype == LLAMA_FTYPE_MOSTLY_Q4_K_M) new_type = GGML_TYPE_Q5_K;651 else if (ftype == LLAMA_FTYPE_MOSTLY_Q5_K_M) new_type = GGML_TYPE_Q6_K;652 }653 else if (category == tensor_category::FFN_GATE) {654 auto info = layer_info(qs.i_ffn_gate, qs.n_ffn_gate, name.c_str());655 int i_layer = info.first, n_layer = info.second;656 if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) {657 new_type = GGML_TYPE_IQ3_XXS;658 }659 ++qs.i_ffn_gate;660 }661 else if (category == tensor_category::FFN_UP) {662 auto info = layer_info(qs.i_ffn_up, qs.n_ffn_up, name.c_str());663 int i_layer = info.first, n_layer = info.second;664 if (ftype == LLAMA_FTYPE_MOSTLY_IQ3_XS && (i_layer >= n_layer/8 && i_layer < 7*n_layer/8)) {665 new_type = GGML_TYPE_IQ3_XXS;666 }667 ++qs.i_ffn_up;668 }669 670 return new_type;671}672 673// outer wrapper: determine the ggml_type that this tensor should be quantized to674static ggml_type llama_tensor_get_type(quantize_state_impl & qs, const llama_model_quantize_params * params, const ggml_tensor * tensor, ggml_type default_type, const tensor_metadata & tm) {675 if (!tensor_allows_quantization(params, qs.model.arch, tensor)) {676 return tensor->type;677 }678 if (params->token_embedding_type < GGML_TYPE_COUNT && tm.category == tensor_category::TOKEN_EMBD) {679 return params->token_embedding_type;680 }681 if (params->output_tensor_type < GGML_TYPE_COUNT && tm.category == tensor_category::OUTPUT) {682 return params->output_tensor_type;683 }684 685 ggml_type new_type = default_type;686 687 // get more optimal quantization type based on the tensor shape, layer, etc.688 if (ggml_is_quantized(default_type)) {689 // if the user provided tensor types - use those690 bool manual = false;691 if (!qs.tensor_type_patterns.empty()) {692 const std::string tensor_name(tensor->name);693 for (const auto & [pattern, qtype] : qs.tensor_type_patterns) {694 if (std::regex_search(tensor_name, pattern)) {695 if (qtype != new_type) {696 LLAMA_LOG_WARN("%s: %-36s - applying manual override: %s -> %s\n",697 __func__, tensor_name.c_str(), ggml_type_name(new_type), ggml_type_name(qtype));698 new_type = qtype;699 }700 manual = true;701 break;702 }703 }704 }705 706 // if not manual - use the standard logic for choosing the quantization type based on the selected mixture707 if (!manual && !params->pure) {708 new_type = llama_tensor_get_type_impl(qs, new_type, tensor, params->ftype, tm.category);709 }710 711 // incompatible tensor shapes are handled here - fallback to a compatible type712 new_type = tensor_type_fallback(qs, tensor, new_type);713 }714 715 return new_type;716}717 718//719// quantization implementation720//721 722static size_t llama_tensor_quantize_impl(enum ggml_type new_type, const float * f32_data, void * new_data, const int64_t chunk_size, int64_t nrows, int64_t n_per_row, const float * imatrix, std::vector<std::thread> & workers, const int nthread) {723 if (nthread < 2) {724 // single-thread725 size_t new_size = ggml_quantize_chunk(new_type, f32_data, new_data, 0, nrows, n_per_row, imatrix);726 if (!ggml_validate_row_data(new_type, new_data, new_size)) {727 throw std::runtime_error("quantized data validation failed");728 }729 return new_size;730 }731 732 std::mutex mutex;733 int64_t counter = 0;734 size_t new_size = 0;735 bool valid = true;736 auto compute = [&mutex, &counter, &new_size, &valid, new_type, f32_data, new_data, chunk_size,737 nrows, n_per_row, imatrix]() {738 const int64_t nrows_per_chunk = chunk_size / n_per_row;739 size_t local_size = 0;740 while (true) {741 std::unique_lock<std::mutex> lock(mutex);742 int64_t first_row = counter; counter += nrows_per_chunk;743 if (first_row >= nrows) {744 if (local_size > 0) {745 new_size += local_size;746 }747 break;748 }749 lock.unlock();750 const int64_t this_nrow = std::min(nrows - first_row, nrows_per_chunk);751 size_t this_size = ggml_quantize_chunk(new_type, f32_data, new_data, first_row * n_per_row, this_nrow, n_per_row, imatrix);752 local_size += this_size;753 754 // validate the quantized data755 const size_t row_size = ggml_row_size(new_type, n_per_row);756 void * this_data = (char *) new_data + first_row * row_size;757 if (!ggml_validate_row_data(new_type, this_data, this_size)) {758 std::unique_lock<std::mutex> lock(mutex);759 valid = false;760 break;761 }762 }763 };764 for (int it = 0; it < nthread - 1; ++it) {765 workers.emplace_back(compute);766 }767 compute();768 for (auto & w : workers) { w.join(); }769 workers.clear();770 if (!valid) {771 throw std::runtime_error("quantized data validation failed");772 }773 return new_size;774}775 776//777// imatrix requirement check778//779 780static bool tensor_requires_imatrix(const char * tensor_name, const ggml_type dst_type, const llama_ftype ftype) {781 if (tensor_name_match_token_embd(tensor_name) || tensor_name_match_output_weight(tensor_name)) {782 return false;783 }784 switch (dst_type) {785 case GGML_TYPE_IQ3_XXS:786 case GGML_TYPE_IQ2_XXS:787 case GGML_TYPE_IQ2_XS:788 case GGML_TYPE_IQ2_S:789 case GGML_TYPE_IQ1_M:790 case GGML_TYPE_IQ1_S:791 return true;792 case GGML_TYPE_Q2_K:793 // as a general rule, the k-type quantizations don't require imatrix data.794 // the only exception is Q2_K tensors that are part of a Q2_K_S file.795 return ftype == LLAMA_FTYPE_MOSTLY_Q2_K_S;796 default:797 return false;798 }799}800 801//802// given a file type, get the default tensor type803//804 805ggml_type llama_ftype_get_default_type(llama_ftype ftype) {806 switch (ftype) {807 case LLAMA_FTYPE_MOSTLY_Q4_0: return GGML_TYPE_Q4_0;808 case LLAMA_FTYPE_MOSTLY_Q4_1: return GGML_TYPE_Q4_1;809 case LLAMA_FTYPE_MOSTLY_Q5_0: return GGML_TYPE_Q5_0;810 case LLAMA_FTYPE_MOSTLY_Q5_1: return GGML_TYPE_Q5_1;811 case LLAMA_FTYPE_MOSTLY_Q8_0: return GGML_TYPE_Q8_0;812 case LLAMA_FTYPE_MOSTLY_F16: return GGML_TYPE_F16;813 case LLAMA_FTYPE_MOSTLY_BF16: return GGML_TYPE_BF16;814 case LLAMA_FTYPE_ALL_F32: return GGML_TYPE_F32;815 case LLAMA_FTYPE_MOSTLY_Q1_0: return GGML_TYPE_Q1_0;816 case LLAMA_FTYPE_MOSTLY_Q2_0: return GGML_TYPE_Q2_0;817 818 case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return GGML_TYPE_MXFP4;819 820 // K-quants821 case LLAMA_FTYPE_MOSTLY_Q2_K_S:822 case LLAMA_FTYPE_MOSTLY_Q2_K: return GGML_TYPE_Q2_K;823 case LLAMA_FTYPE_MOSTLY_IQ3_XS: return GGML_TYPE_IQ3_S;824 case LLAMA_FTYPE_MOSTLY_Q3_K_S:825 case LLAMA_FTYPE_MOSTLY_Q3_K_M:826 case LLAMA_FTYPE_MOSTLY_Q3_K_L: return GGML_TYPE_Q3_K;827 case LLAMA_FTYPE_MOSTLY_Q4_K_S:828 case LLAMA_FTYPE_MOSTLY_Q4_K_M: return GGML_TYPE_Q4_K;829 case LLAMA_FTYPE_MOSTLY_Q5_K_S:830 case LLAMA_FTYPE_MOSTLY_Q5_K_M: return GGML_TYPE_Q5_K;831 case LLAMA_FTYPE_MOSTLY_Q6_K: return GGML_TYPE_Q6_K;832 case LLAMA_FTYPE_MOSTLY_TQ1_0: return GGML_TYPE_TQ1_0;833 case LLAMA_FTYPE_MOSTLY_TQ2_0: return GGML_TYPE_TQ2_0;834 case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return GGML_TYPE_IQ2_XXS;835 case LLAMA_FTYPE_MOSTLY_IQ2_XS: return GGML_TYPE_IQ2_XS;836 case LLAMA_FTYPE_MOSTLY_IQ2_S: return GGML_TYPE_IQ2_XS;837 case LLAMA_FTYPE_MOSTLY_IQ2_M: return GGML_TYPE_IQ2_S;838 case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return GGML_TYPE_IQ3_XXS;839 case LLAMA_FTYPE_MOSTLY_IQ1_S: return GGML_TYPE_IQ1_S;840 case LLAMA_FTYPE_MOSTLY_IQ1_M: return GGML_TYPE_IQ1_M;841 case LLAMA_FTYPE_MOSTLY_IQ4_NL: return GGML_TYPE_IQ4_NL;842 case LLAMA_FTYPE_MOSTLY_IQ4_XS: return GGML_TYPE_IQ4_XS;843 case LLAMA_FTYPE_MOSTLY_IQ3_S:844 case LLAMA_FTYPE_MOSTLY_IQ3_M: return GGML_TYPE_IQ3_S;845 846 default: return GGML_TYPE_COUNT;847 }848}849 850 851static void init_quantize_state_counters(quantize_state_impl & qs, std::vector<tensor_metadata> & metadata) {852 for (auto & tm : metadata) {853 tensor_category cat = tensor_get_category(tm.name);854 tm.category = cat;855 856 if (category_is_attn_v(cat)) {857 ++qs.n_attention_wv;858 }859 860 if (cat == tensor_category::OUTPUT) {861 qs.has_tied_embeddings = false;862 }863 }864 qs.n_ffn_down = qs.n_ffn_gate = qs.n_ffn_up = (int)qs.model.hparams.n_layer_all;865}866 867//868// main quantization driver869//870 871static void llama_model_quantize_impl(const std::string & fname_inp, const std::string & fname_out, const llama_model_quantize_params * params) {872 llama_ftype ftype = params->ftype;873 874 int nthread = params->nthread;875 876 if (nthread <= 0) {877 nthread = std::thread::hardware_concurrency();878 }879 880 ggml_type default_type = llama_ftype_get_default_type(ftype);881 if (default_type == GGML_TYPE_COUNT) {882 throw std::runtime_error(format("invalid output file type %d\n", ftype));883 }884 885 // mmap consistently increases speed on Linux, and also increases speed on Windows with886 // hot cache. It may cause a slowdown on macOS, possibly related to free memory.887#if defined(__linux__) || defined(_WIN32)888 constexpr llama_load_mode load_mode = LLAMA_LOAD_MODE_MMAP;889#else890 constexpr llama_load_mode load_mode = LLAMA_LOAD_MODE_NONE;891#endif892 893 const llama_model_kv_override * kv_overrides = params->kv_overrides;894 std::vector<std::string> splits = {};895 llama_model_loader ml(/*metadata*/ nullptr, /*set_tensor_data*/ nullptr, /*set_tensor_data_ud*/ nullptr,896 fname_inp, splits, /*file*/ nullptr, /*load_mode*/ load_mode, /*check_tensors*/ true, /*no_alloc*/ false, /*load_mtp*/ true, kv_overrides, nullptr);897 ml.init_mappings(false); // no prefetching898 899 auto mparams = llama_model_default_params();900 std::unique_ptr<llama_model> model_ptr(llama_model_create(ml, mparams));901 902 auto * model = dynamic_cast<llama_model_base *>(model_ptr.get());903 if (model == nullptr) {904 GGML_ABORT("fatal error: model does not implement llama_model_base");905 }906 907 model->load_hparams(ml);908 model->load_stats (ml);909 910 quantize_state_impl qs(*model, params);911 912 if (params->only_copy) {913 ftype = ml.ftype;914 }915 std::unordered_map<std::string, std::vector<float>> i_data;916 const std::unordered_map<std::string, std::vector<float>> * imatrix_data = nullptr;917 if (params->imatrix) {918 for (const llama_model_imatrix_data * p = params->imatrix; p->name != nullptr; p++) {919 i_data.emplace(p->name, std::vector<float>(p->data, p->data + p->size));920 }921 imatrix_data = & i_data;922 if (imatrix_data) {923 LLAMA_LOG_INFO("\n%s: have importance matrix data with %d entries\n",924 __func__, (int)imatrix_data->size());925 qs.has_imatrix = true;926 // check imatrix for nans or infs927 for (const auto & kv : *imatrix_data) {928 for (float f : kv.second) {929 if (!std::isfinite(f)) {930 throw std::runtime_error(format("imatrix contains non-finite value %f\n", f));931 }932 }933 }934 }935 }936 937 const size_t align = GGUF_DEFAULT_ALIGNMENT;938 gguf_context_ptr ctx_out { gguf_init_empty() };939 940 std::vector<int> prune_list = {};941 if (params->prune_layers) {942 for (const int32_t * p = params->prune_layers; * p != -1; p++) {943 prune_list.push_back(* p);944 }945 }946 947 // copy the KV pairs from the input file948 gguf_set_kv (ctx_out.get(), ml.metadata);949 gguf_set_val_u32(ctx_out.get(), ml.llm_kv(LLM_KV_GENERAL_QUANTIZATION_VERSION).c_str(), GGML_QNT_VERSION);950 gguf_set_val_u32(ctx_out.get(), ml.llm_kv(LLM_KV_GENERAL_FILE_TYPE).c_str(), ftype);951 952 // Remove split metadata953 gguf_remove_key(ctx_out.get(), ml.llm_kv(LLM_KV_SPLIT_NO).c_str());954 gguf_remove_key(ctx_out.get(), ml.llm_kv(LLM_KV_SPLIT_COUNT).c_str());955 gguf_remove_key(ctx_out.get(), ml.llm_kv(LLM_KV_SPLIT_TENSORS_COUNT).c_str());956 957 if (params->kv_overrides) {958 for (const llama_model_kv_override * o = params->kv_overrides; o->key[0] != 0; ++o) {959 if (o->tag == LLAMA_KV_OVERRIDE_TYPE_FLOAT) {960 gguf_set_val_f32(ctx_out.get(), o->key, o->val_f64);961 } else if (o->tag == LLAMA_KV_OVERRIDE_TYPE_INT) {962 // Setting type to UINT32. See https://github.com/ggml-org/llama.cpp/pull/14182 for context963 gguf_set_val_u32(ctx_out.get(), o->key, (uint32_t)std::abs(o->val_i64));964 } else if (o->tag == LLAMA_KV_OVERRIDE_TYPE_BOOL) {965 gguf_set_val_bool(ctx_out.get(), o->key, o->val_bool);966 } else if (o->tag == LLAMA_KV_OVERRIDE_TYPE_STR) {967 gguf_set_val_str(ctx_out.get(), o->key, o->val_str);968 } else {969 LLAMA_LOG_WARN("%s: unknown KV override type for key %s\n", __func__, o->key);970 }971 }972 }973 974 std::map<int, std::string> mapped;975 int blk_id = 0;976 977 // make a list of weights978 std::vector<const llama_model_loader::llama_tensor_weight *> tensors;979 tensors.reserve(ml.weights_map.size());980 for (const auto & it : ml.weights_map) {981 const std::string remapped_name(remap_layer(it.first, prune_list, mapped, blk_id));982 if (remapped_name.empty()) {983 LLAMA_LOG_DEBUG("%s: pruning tensor %s\n", __func__, it.first.c_str());984 continue;985 }986 987 if (remapped_name != it.first) {988 ggml_set_name(it.second.tensor, remapped_name.c_str());989 LLAMA_LOG_DEBUG("%s: tensor %s remapped to %s\n", __func__, it.first.c_str(), ggml_get_name(it.second.tensor));990 }991 tensors.push_back(&it.second);992 }993 if (!prune_list.empty()) {994 gguf_set_val_u32(ctx_out.get(), ml.llm_kv(LLM_KV_BLOCK_COUNT).c_str(), blk_id);995 }996 997 // keep_split requires that the weights are sorted by split index998 if (params->keep_split) {999 std::sort(tensors.begin(), tensors.end(), [](const llama_model_loader::llama_tensor_weight * a, const llama_model_loader::llama_tensor_weight * b) {1000 if (a->idx == b->idx) {1001 return a->offs < b->offs;1002 }1003 return a->idx < b->idx;1004 });1005 }1006 1007 // compute tensor metadata once and cache it1008 std::vector<tensor_metadata> metadata(tensors.size());1009 for (size_t i = 0; i < tensors.size(); ++i) {1010 metadata[i].name = ggml_get_name(tensors[i]->tensor);1011 }1012 1013 // initialize quantization state counters and metadata categories1014 init_quantize_state_counters(qs, metadata);1015 1016 int idx = 0;1017 uint16_t n_split = 1;1018 1019 // Assume split index is continuous1020 if (params->keep_split) {1021 for (const auto * it : tensors) {1022 n_split = std::max(uint16_t(it->idx + 1), n_split);1023 }1024 }1025 std::vector<gguf_context_ptr> ctx_outs(n_split);1026 ctx_outs[0] = std::move(ctx_out);1027 1028 // flag for --dry-run1029 bool will_require_imatrix = false;1030 1031 //1032 // preliminary iteration over all weights1033 //1034 1035 for (size_t i = 0; i < tensors.size(); ++i) {1036 const auto * it = tensors[i];1037 const struct ggml_tensor * tensor = it->tensor;1038 1039 uint16_t i_split = params->keep_split ? it->idx : 0;1040 if (!ctx_outs[i_split]) {1041 ctx_outs[i_split].reset(gguf_init_empty());1042 }1043 gguf_add_tensor(ctx_outs[i_split].get(), tensor);1044 1045 metadata[i].allows_quantization = tensor_allows_quantization(params, model->arch, tensor);1046 1047 if (metadata[i].allows_quantization) {1048 metadata[i].target_type = llama_tensor_get_type(qs, params, tensor, default_type, metadata[i]);1049 } else {1050 metadata[i].target_type = tensor->type;1051 }1052 1053 metadata[i].requires_imatrix = tensor_requires_imatrix(tensor->name, metadata[i].target_type, ftype);1054 1055 if (params->imatrix) {1056 metadata[i].remapped_imatrix_name = remap_imatrix(tensor->name, mapped);1057 } else if (metadata[i].allows_quantization && metadata[i].requires_imatrix) {1058 if (params->dry_run) {1059 will_require_imatrix = true;1060 } else {1061 LLAMA_LOG_ERROR("\n============================================================================\n"1062 " ERROR: this quantization requires an importance matrix!\n"1063 " - offending tensor: %s\n"1064 " - target type: %s\n"1065 "============================================================================\n\n",1066 metadata[i].name.c_str(), ggml_type_name(metadata[i].target_type));1067 throw std::runtime_error("this quantization requires an imatrix!");1068 }1069 }1070 }1071 1072 // Set split info if needed1073 if (n_split > 1) {1074 for (size_t i = 0; i < ctx_outs.size(); ++i) {1075 gguf_set_val_u16(ctx_outs[i].get(), ml.llm_kv(LLM_KV_SPLIT_NO).c_str(), i);1076 gguf_set_val_u16(ctx_outs[i].get(), ml.llm_kv(LLM_KV_SPLIT_COUNT).c_str(), n_split);1077 gguf_set_val_i32(ctx_outs[i].get(), ml.llm_kv(LLM_KV_SPLIT_TENSORS_COUNT).c_str(), (int32_t)tensors.size());1078 }1079 }1080 1081 size_t total_size_org = 0;1082 size_t total_size_new = 0;1083 1084 std::vector<std::thread> workers;1085 workers.reserve(nthread);1086 1087 std::vector<no_init<uint8_t>> read_data;1088 std::vector<no_init<uint8_t>> work;1089 std::vector<no_init<float>> f32_conv_buf;1090 1091 int cur_split = -1;1092 std::ofstream fout;1093 auto close_ofstream = [&]() {1094 // Write metadata and close file handler1095 if (fout.is_open()) {1096 fout.seekp(0);1097 std::vector<uint8_t> data(gguf_get_meta_size(ctx_outs[cur_split].get()));1098 gguf_get_meta_data(ctx_outs[cur_split].get(), data.data());1099 fout.write((const char *) data.data(), data.size());1100 fout.close();1101 }1102 };1103 auto new_ofstream = [&](int index) {1104 cur_split = index;1105 GGML_ASSERT(ctx_outs[cur_split] && "Find uninitialized gguf_context");1106 std::string fname = fname_out;1107 if (params->keep_split) {1108 std::vector<char> split_path(llama_path_max(), 0);1109 llama_split_path(split_path.data(), split_path.size(), fname_out.c_str(), cur_split, n_split);1110 fname = std::string(split_path.data());1111 }1112 1113 fout = std::ofstream(fname, std::ios::binary);1114 fout.exceptions(std::ofstream::failbit); // fail fast on write errors1115 const size_t meta_size = gguf_get_meta_size(ctx_outs[cur_split].get());1116 // placeholder for the meta data1117 ::zeros(fout, meta_size);1118 };1119 1120 // no output file for --dry-run1121 if (!params->dry_run) {1122 new_ofstream(0);1123 }1124 1125 //1126 // main loop: iterate over all weights1127 //1128 1129 for (size_t i = 0; i < tensors.size(); ++i) {1130 const auto & weight = *tensors[i];1131 const auto & tm = metadata[i];1132 ggml_tensor * tensor = weight.tensor;1133 1134 if (!params->dry_run && (weight.idx != cur_split && params->keep_split)) {1135 close_ofstream();1136 new_ofstream(weight.idx);1137 }1138 1139 const size_t tensor_size = ggml_nbytes(tensor);1140 1141 if (!params->dry_run) {1142 if (!ml.use_mmap) {1143 if (read_data.size() < tensor_size) {1144 read_data.resize(tensor_size);1145 }1146 tensor->data = read_data.data();1147 }1148 ml.load_data_for(tensor);1149 }1150 1151 LLAMA_LOG_INFO("[%4d/%4d] %-36s - [%s], type = %6s, ",1152 ++idx, ml.n_tensors,1153 ggml_get_name(tensor),1154 llama_format_tensor_shape(tensor).c_str(),1155 ggml_type_name(tensor->type));1156 1157 const ggml_type cur_type = tensor->type;1158 const ggml_type new_type = tm.target_type;1159 1160 // If we've decided to quantize to the same type the tensor is already1161 // in then there's nothing to do.1162 bool quantize = cur_type != new_type;1163 1164 void * new_data;1165 size_t new_size;1166 1167 if (params->dry_run) {1168 // the --dry-run option calculates the final quantization size without quantizing1169 if (quantize) {1170 new_size = ggml_nrows(tensor) * ggml_row_size(new_type, tensor->ne[0]);1171 LLAMA_LOG_INFO("size = %8.2f MiB -> %8.2f MiB (%s)\n",1172 tensor_size/1024.0/1024.0,1173 new_size/1024.0/1024.0,1174 ggml_type_name(new_type));1175 if (!will_require_imatrix && tm.requires_imatrix) {1176 will_require_imatrix = true;1177 }1178 } else {1179 new_size = tensor_size;1180 LLAMA_LOG_INFO("size = %8.3f MiB\n", new_size/1024.0/1024.0);1181 }1182 total_size_org += tensor_size;1183 total_size_new += new_size;1184 continue;1185 } else {1186 // no --dry-run, perform quantization1187 if (!quantize) {1188 new_data = tensor->data;1189 new_size = tensor_size;1190 LLAMA_LOG_INFO("size = %8.3f MiB\n", tensor_size/1024.0/1024.0);1191 } else {1192 const int64_t nelements = ggml_nelements(tensor);1193 1194 const float * imatrix = nullptr;1195 if (imatrix_data) {1196 auto it = imatrix_data->find(tm.remapped_imatrix_name);1197 if (it == imatrix_data->end()) {1198 LLAMA_LOG_INFO("\n====== %s: did not find weights for %s\n", __func__, tensor->name);1199 } else {1200 if (it->second.size() == (size_t)tensor->ne[0]*tensor->ne[2]) {