Brunobkr/llama.cpp_AlgMor24_github

1#include "llama.h"2 3#include "build-info.h"4#include "common.h"5#include "imatrix-loader.h"6 7#include "gguf.h"8 9#include <algorithm>10#include <cctype>11#include <clocale>12#include <cmath>13#include <cstdio>14#include <cstring>15#include <vector>16#include <string>17#include <unordered_map>18#include <fstream>19#include <filesystem>20 21// result of parsing --tensor-type option22// changes to this struct must also be reflected in src/llama-quant.cpp23struct tensor_type_option {24    std::string name;25    ggml_type type = GGML_TYPE_COUNT;26};27 28struct quant_option {29    std::string name;30    llama_ftype ftype;31    std::string desc;32};33 34static const std::vector<quant_option> QUANT_OPTIONS = {35    { "Q1_0",     LLAMA_FTYPE_MOSTLY_Q1_0,     " 1.125 bpw quantization",           },36    { "Q2_0",     LLAMA_FTYPE_MOSTLY_Q2_0,     " 2.25 bpw quantization (group 64)",  },37    { "Q4_0",     LLAMA_FTYPE_MOSTLY_Q4_0,     " 4.34G, +0.4685 ppl @ Llama-3-8B",  },38    { "Q4_1",     LLAMA_FTYPE_MOSTLY_Q4_1,     " 4.78G, +0.4511 ppl @ Llama-3-8B",  },39    { "MXFP4_MOE",LLAMA_FTYPE_MOSTLY_MXFP4_MOE," MXFP4 MoE",  },40    { "Q5_0",     LLAMA_FTYPE_MOSTLY_Q5_0,     " 5.21G, +0.1316 ppl @ Llama-3-8B",  },41    { "Q5_1",     LLAMA_FTYPE_MOSTLY_Q5_1,     " 5.65G, +0.1062 ppl @ Llama-3-8B",  },42    { "IQ2_XXS",  LLAMA_FTYPE_MOSTLY_IQ2_XXS,  " 2.06 bpw quantization",            },43    { "IQ2_XS",   LLAMA_FTYPE_MOSTLY_IQ2_XS,   " 2.31 bpw quantization",            },44    { "IQ2_S",    LLAMA_FTYPE_MOSTLY_IQ2_S,    " 2.5  bpw quantization",            },45    { "IQ2_M",    LLAMA_FTYPE_MOSTLY_IQ2_M,    " 2.7  bpw quantization",            },46    { "IQ1_S",    LLAMA_FTYPE_MOSTLY_IQ1_S,    " 1.56 bpw quantization",            },47    { "IQ1_M",    LLAMA_FTYPE_MOSTLY_IQ1_M,    " 1.75 bpw quantization",            },48    { "TQ1_0",    LLAMA_FTYPE_MOSTLY_TQ1_0,    " 1.69 bpw ternarization",           },49    { "TQ2_0",    LLAMA_FTYPE_MOSTLY_TQ2_0,    " 2.06 bpw ternarization",           },50    { "Q2_K",     LLAMA_FTYPE_MOSTLY_Q2_K,     " 2.96G, +3.5199 ppl @ Llama-3-8B",  },51    { "Q2_K_S",   LLAMA_FTYPE_MOSTLY_Q2_K_S,   " 2.96G, +3.1836 ppl @ Llama-3-8B",  },52    { "IQ3_XXS",  LLAMA_FTYPE_MOSTLY_IQ3_XXS,  " 3.06 bpw quantization",            },53    { "IQ3_S",    LLAMA_FTYPE_MOSTLY_IQ3_S,    " 3.44 bpw quantization",            },54    { "IQ3_M",    LLAMA_FTYPE_MOSTLY_IQ3_M,    " 3.66 bpw quantization mix",        },55    { "Q3_K",     LLAMA_FTYPE_MOSTLY_Q3_K_M,   "alias for Q3_K_M"                   },56    { "IQ3_XS",   LLAMA_FTYPE_MOSTLY_IQ3_XS,   " 3.3 bpw quantization",             },57    { "Q3_K_S",   LLAMA_FTYPE_MOSTLY_Q3_K_S,   " 3.41G, +1.6321 ppl @ Llama-3-8B",  },58    { "Q3_K_M",   LLAMA_FTYPE_MOSTLY_Q3_K_M,   " 3.74G, +0.6569 ppl @ Llama-3-8B",  },59    { "Q3_K_L",   LLAMA_FTYPE_MOSTLY_Q3_K_L,   " 4.03G, +0.5562 ppl @ Llama-3-8B",  },60    { "IQ4_NL",   LLAMA_FTYPE_MOSTLY_IQ4_NL,   " 4.50 bpw non-linear quantization", },61    { "IQ4_XS",   LLAMA_FTYPE_MOSTLY_IQ4_XS,   " 4.25 bpw non-linear quantization", },62    { "Q4_K",     LLAMA_FTYPE_MOSTLY_Q4_K_M,   "alias for Q4_K_M",                  },63    { "Q4_K_S",   LLAMA_FTYPE_MOSTLY_Q4_K_S,   " 4.37G, +0.2689 ppl @ Llama-3-8B",  },64    { "Q4_K_M",   LLAMA_FTYPE_MOSTLY_Q4_K_M,   " 4.58G, +0.1754 ppl @ Llama-3-8B",  },65    { "Q5_K",     LLAMA_FTYPE_MOSTLY_Q5_K_M,   "alias for Q5_K_M",                  },66    { "Q5_K_S",   LLAMA_FTYPE_MOSTLY_Q5_K_S,   " 5.21G, +0.1049 ppl @ Llama-3-8B",  },67    { "Q5_K_M",   LLAMA_FTYPE_MOSTLY_Q5_K_M,   " 5.33G, +0.0569 ppl @ Llama-3-8B",  },68    { "Q6_K",     LLAMA_FTYPE_MOSTLY_Q6_K,     " 6.14G, +0.0217 ppl @ Llama-3-8B",  },69    { "Q8_0",     LLAMA_FTYPE_MOSTLY_Q8_0,     " 7.96G, +0.0026 ppl @ Llama-3-8B",  },70    { "F16",      LLAMA_FTYPE_MOSTLY_F16,      "14.00G, +0.0020 ppl @ Mistral-7B",  },71    { "BF16",     LLAMA_FTYPE_MOSTLY_BF16,     "14.00G, -0.0050 ppl @ Mistral-7B",  },72    { "F32",      LLAMA_FTYPE_ALL_F32,         "26.00G              @ 7B",          },73    // Note: Ensure COPY comes after F32 to avoid ftype 0 from matching.74    { "COPY",     LLAMA_FTYPE_ALL_F32,         "only copy tensors, no quantizing",  },75};76 77static const char * const LLM_KV_QUANTIZE_IMATRIX_FILE       = "quantize.imatrix.file";78static const char * const LLM_KV_QUANTIZE_IMATRIX_DATASET    = "quantize.imatrix.dataset";79static const char * const LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES  = "quantize.imatrix.entries_count";80static const char * const LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS   = "quantize.imatrix.chunks_count";81 82static bool striequals(const char * a, const char * b) {83    while (*a && *b) {84        if (std::tolower(*a) != std::tolower(*b)) {85            return false;86        }87        a++; b++;88    }89    return *a == *b;90}91 92static bool try_parse_ftype(const std::string & ftype_str_in, llama_ftype & ftype, std::string & ftype_str_out) {93    std::string ftype_str;94 95    for (auto ch : ftype_str_in) {96        ftype_str.push_back(std::toupper(ch));97    }98    for (const auto & it : QUANT_OPTIONS) {99        if (striequals(it.name.c_str(), ftype_str.c_str())) {100            ftype = it.ftype;101            ftype_str_out = it.name;102            return true;103        }104    }105    try {106        int ftype_int = std::stoi(ftype_str);107        for (const auto & it : QUANT_OPTIONS) {108            if (it.ftype == ftype_int) {109                ftype = it.ftype;110                ftype_str_out = it.name;111                return true;112            }113        }114    }115    catch (...) {116        // stoi failed117    }118    return false;119}120 121[[noreturn]]122static void usage(const char * executable) {123    printf("usage: %s [--help] [--allow-requantize] [--leave-output-tensor] [--pure] [--imatrix] [--include-weights]\n", executable);124    printf("       [--exclude-weights] [--output-tensor-type] [--token-embedding-type] [--tensor-type] [--tensor-type-file]\n");125    printf("       [--prune-layers] [--keep-split] [--override-kv] [--dry-run]\n");126    printf("       model-f32.gguf [model-quant.gguf] type [nthreads]\n\n");127    printf("  --allow-requantize\n");128    printf("                                      allow requantizing tensors that have already been quantized\n");129    printf("                                      WARNING: this can severely reduce quality compared to quantizing\n");130    printf("                                               from 16bit or 32bit!\n");131    printf("  --leave-output-tensor\n");132    printf("                                      leave output.weight un(re)quantized\n");133    printf("                                      increases model size but may also increase quality, especially when requantizing\n");134    printf("  --pure\n");135    printf("                                      disable k-quant mixtures and quantize all tensors to the same type\n");136    printf("  --imatrix file_name\n");137    printf("                                      use data in file_name as importance matrix for quant optimizations\n");138    printf("  --include-weights tensor_name\n");139    printf("                                      use importance matrix for this/these tensor(s)\n");140    printf("  --exclude-weights tensor_name\n");141    printf("                                      do not use importance matrix for this/these tensor(s)\n");142    printf("  --output-tensor-type ggml_type\n");143    printf("                                      use this ggml_type for the output.weight tensor\n");144    printf("  --token-embedding-type ggml_type\n");145    printf("                                      use this ggml_type for the token embeddings tensor\n");146    printf("  --tensor-type tensor_name=ggml_type\n");147    printf("                                      quantize this tensor to this ggml_type\n");148    printf("                                      this is an advanced option to selectively quantize tensors. may be specified multiple times.\n");149    printf("                                      example: --tensor-type attn_q=q8_0\n");150    printf("  --tensor-type-file tensor_types.txt\n");151    printf("                                      list of tensors to quantize to a specific ggml_type\n");152    printf("                                      this is an advanced option to selectively quantize a long list of tensors.\n");153    printf("                                      the file should use the same format as above, separated by spaces or newlines.\n");154    printf("  --prune-layers L0,L1,L2...\n");155    printf("                                      comma-separated list of layer numbers to prune from the model\n");156    printf("                                      WARNING: this is an advanced option, use with care.\n");157    printf("  --keep-split\n");158    printf("                                      generate quantized model in the same shards as input\n");159    printf("  --override-kv KEY=TYPE:VALUE\n");160    printf("                                      override model metadata by key in the quantized model. may be specified multiple times.\n");161    printf("                                      WARNING: this is an advanced option, use with care.\n");162    printf("  --dry-run\n");163    printf("                                      calculate and show the final quantization size without performing quantization\n");164    printf("                                      example: llama-quantize --dry-run model-f32.gguf Q4_K\n\n");165    printf("note: --include-weights and --exclude-weights cannot be used together\n\n");166    printf("-----------------------------------------------------------------------------\n");167    printf(" allowed quantization types\n");168    printf("-----------------------------------------------------------------------------\n\n");169    for (const auto & it : QUANT_OPTIONS) {170        if (it.name != "COPY") {171            printf("  %2d  or  ", it.ftype);172        } else {173            printf("          ");174        }175        printf("%-7s : %s\n", it.name.c_str(), it.desc.c_str());176    }177    exit(1);178}179 180static int load_imatrix(const std::string & imatrix_file, std::vector<std::string> & imatrix_datasets, std::unordered_map<std::string, std::vector<float>> & imatrix_data) {181    common_imatrix loaded;182    if (!common_imatrix_load(imatrix_file, loaded)) {183        fprintf(stderr, "%s: failed to load imatrix from '%s'\n", __func__, imatrix_file.c_str());184        exit(1);185    }186 187    if (!loaded.is_legacy && !loaded.has_metadata) {188        fprintf(stderr, "%s: missing imatrix metadata in file %s\n", __func__, imatrix_file.c_str());189        exit(1);190    }191 192    for (const auto & [name, entry] : loaded.entries) {193        auto & e = imatrix_data[name];194        e.resize(entry.sums.size());195 196        if (!loaded.is_legacy) {197            // GGUF format: normalize by per-expert counts198            const int64_t ncounts = entry.counts.size();199            const int64_t ne0     = (int64_t) entry.sums.size() / ncounts;200 201            for (int64_t j = 0; j < ncounts; ++j) {202                const float count = (float) entry.counts[j];203                if (count > 0.0f) {204                    for (int64_t i = 0; i < ne0; ++i) {205                        e[j*ne0 + i] = entry.sums[j*ne0 + i] / count;206                    }207                } else {208                    for (int64_t i = 0; i < ne0; ++i) {209                        e[j*ne0 + i] = 1;210                    }211                }212            }213 214            if (getenv("LLAMA_TRACE")) {215                float max_count = 0.0f;216                for (int64_t j = 0; j < ncounts; ++j) {217                    const float count = (float) entry.counts[j];218                    if (count > max_count) {219                        max_count = count;220                    }221                }222                printf("%s: loaded data (size = %6d, n_tokens = %6d, n_chunks = %6d) for '%s'\n",223                       __func__, int(e.size()), int(max_count), int(max_count / loaded.chunk_size), name.c_str());224            }225        } else {226            // Legacy format: sums contain (raw/count)*ncall, divide by ncall227            const int64_t ncall = entry.counts.empty() ? 0 : entry.counts[0];228            if (ncall > 0) {229                for (size_t i = 0; i < entry.sums.size(); ++i) {230                    e[i] = entry.sums[i] / ncall;231                }232            } else {233                for (size_t i = 0; i < entry.sums.size(); ++i) {234                    e[i] = entry.sums[i];235                }236            }237 238            if (getenv("LLAMA_TRACE")) {239                printf("%s: loaded data (size = %6d, ncall = %6d) for '%s'\n",240                       __func__, int(e.size()), int(ncall), name.c_str());241            }242        }243    }244 245    imatrix_datasets = std::move(loaded.datasets);246 247    if (!imatrix_datasets.empty()) {248        printf("%s: imatrix datasets=['%s'", __func__, imatrix_datasets[0].c_str());249        for (size_t i = 1; i < imatrix_datasets.size(); ++i) {250            printf(", '%s'", imatrix_datasets[i].c_str());251        }252        printf("]\n");253    }254 255    printf("%s: loaded %d importance matrix entries from %s computed on %d chunks\n", __func__, int(imatrix_data.size()), imatrix_file.c_str(), loaded.chunk_count);256 257    return loaded.chunk_count;258}259 260static int prepare_imatrix(const std::string & imatrix_file,261        std::vector<std::string> & imatrix_dataset,262        const std::vector<std::string> & included_weights,263        const std::vector<std::string> & excluded_weights,264        std::unordered_map<std::string, std::vector<float>> & imatrix_data) {265    int m_last_call = -1;266    if (!imatrix_file.empty()) {267        m_last_call = load_imatrix(imatrix_file, imatrix_dataset, imatrix_data);268    }269    if (imatrix_data.empty()) {270        return m_last_call;271    }272    if (!excluded_weights.empty()) {273        for (const auto & name : excluded_weights) {274            for (auto it = imatrix_data.begin(); it != imatrix_data.end();) {275                auto pos = it->first.find(name);276                if (pos != std::string::npos) {277                    it = imatrix_data.erase(it);278                } else {279                    ++it;280                }281            }282        }283    }284    if (!included_weights.empty()) {285        std::unordered_map<std::string, std::vector<float>> tmp;286        for (const auto & name : included_weights) {287            for (auto & e : imatrix_data) {288                auto pos = e.first.find(name);289                if (pos != std::string::npos) {290                    tmp.emplace(std::move(e));291                }292            }293        }294        imatrix_data = std::move(tmp);295    }296    if (!imatrix_data.empty()) {297        printf("%s: have %d importance matrix entries\n", __func__, int(imatrix_data.size()));298    }299    return m_last_call;300}301 302static ggml_type parse_ggml_type(const char * arg) {303    for (int i = 0; i < GGML_TYPE_COUNT; ++i) {304        auto type = (ggml_type)i;305        const auto * name = ggml_type_name(type);306        if (name && striequals(name, arg)) {307            return type;308        }309    }310    fprintf(stderr, "\n%s: invalid ggml_type '%s'\n\n", __func__, arg);311    return GGML_TYPE_COUNT;312}313 314static bool parse_tensor_type(const char * data, std::vector<tensor_type_option> & tensor_type) {315    const char * sep = strchr(data, '=');316    if (sep == nullptr) {317        printf("\n%s: malformed tensor type '%s'\n\n", __func__, data);318        return false;319    }320 321    const size_t tn_len = sep - data;322    if (tn_len == 0) {323        printf("\n%s: missing tensor name\n\n", __func__);324        return false;325    }326    if (const size_t qt_len = strlen(sep); qt_len == 1) {327        printf("\n%s: missing quantization type\n\n", __func__);328        return false;329    }330 331    std::string tn(data, tn_len);332    std::transform(tn.begin(), tn.end(), tn.begin(), tolower);333    sep++;334    tensor_type_option tensor_type_opt;335    tensor_type_opt.name = tn;336    tensor_type_opt.type = parse_ggml_type(sep);337    tensor_type.emplace_back(std::move(tensor_type_opt));338    if (tensor_type_opt.type == GGML_TYPE_COUNT) {339        printf("\n%s: invalid quantization type '%s'\n\n", __func__, sep);340        return false;341    }342 343    return true;344}345 346static bool parse_tensor_type_file(const char * filename, std::vector<tensor_type_option> & tensor_type) {347    std::ifstream file(filename);348    if (!file) {349        printf("\n%s: failed to open file '%s': %s\n\n", __func__, filename, std::strerror(errno));350        return false;351    }352 353    std::string arg;354    while (file >> arg) {355        if (!parse_tensor_type(arg.c_str(), tensor_type)) {356            return false;357        }358    }359 360    return true;361}362 363static bool parse_layer_prune(const char * data, std::vector<int> & prune_layers) {364    if (!data) {365        printf("\n%s: no layer pruning ids provided\n\n", __func__);366        return false;367    }368 369    const auto block_ids = string_split<std::string>(data, ',');370    for (const auto & block_id : block_ids) {371        int id;372        try {373            id = std::stoi(block_id);374        } catch (...) {375            id = -1;376        }377        if (id < 0) {378            printf("\n%s: invalid layer id '%s'\n\n", __func__, block_id.c_str());379            return false;380        }381        prune_layers.emplace_back(id);382    }383 384    sort(prune_layers.begin(), prune_layers.end());385    prune_layers.erase(std::unique(prune_layers.begin(), prune_layers.end()), prune_layers.end());386    return true;387}388 389// satisfies -Wmissing-declarations390int llama_quantize(int argc, char ** argv);391 392int llama_quantize(int argc, char ** argv) {393    std::setlocale(LC_NUMERIC, "C");394    if (argc < 3) {395        usage(argv[0]);396    }397 398    llama_model_quantize_params params = llama_model_quantize_default_params();399 400    int arg_idx = 1;401    std::string imatrix_file;402    std::vector<std::string> included_weights, excluded_weights;403    std::vector<llama_model_kv_override> kv_overrides;404    std::vector<tensor_type_option> tensor_type_opts;405    std::vector<int> prune_layers;406 407    for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) {408        if (strcmp(argv[arg_idx], "--leave-output-tensor") == 0) {409            params.quantize_output_tensor = false;410        } else if (strcmp(argv[arg_idx], "--output-tensor-type") == 0) {411            if (arg_idx < argc-1) {412                params.output_tensor_type = parse_ggml_type(argv[++arg_idx]);413                if (params.output_tensor_type == GGML_TYPE_COUNT) {414                    usage(argv[0]);415                }416            } else {417                usage(argv[0]);418            }419        } else if (strcmp(argv[arg_idx], "--token-embedding-type") == 0) {420            if (arg_idx < argc-1) {421                params.token_embedding_type = parse_ggml_type(argv[++arg_idx]);422                if (params.token_embedding_type == GGML_TYPE_COUNT) {423                    usage(argv[0]);424                }425            } else {426                usage(argv[0]);427            }428        } else if (strcmp(argv[arg_idx], "--tensor-type") == 0) {429            if (arg_idx == argc-1 || !parse_tensor_type(argv[++arg_idx], tensor_type_opts)) {430                usage(argv[0]);431            }432        } else if (strcmp(argv[arg_idx], "--tensor-type-file") == 0) {433            if (arg_idx == argc-1 || !parse_tensor_type_file(argv[++arg_idx], tensor_type_opts)) {434                usage(argv[0]);435            }436        } else if (strcmp(argv[arg_idx], "--prune-layers") == 0) {437            if (arg_idx == argc-1 || !parse_layer_prune(argv[++arg_idx], prune_layers)) {438                usage(argv[0]);439            }440        } else if (strcmp(argv[arg_idx], "--override-kv") == 0) {441            if (arg_idx == argc-1 || !string_parse_kv_override(argv[++arg_idx], kv_overrides)) {442                usage(argv[0]);443            }444        } else if (strcmp(argv[arg_idx], "--dry-run") == 0) {445            params.dry_run = true;446        } else if (strcmp(argv[arg_idx], "--allow-requantize") == 0) {447            params.allow_requantize = true;448        } else if (strcmp(argv[arg_idx], "--pure") == 0) {449            params.pure = true;450        } else if (strcmp(argv[arg_idx], "--imatrix") == 0) {451            if (arg_idx < argc-1) {452                imatrix_file = argv[++arg_idx];453            } else {454                usage(argv[0]);455            }456        } else if (strcmp(argv[arg_idx], "--include-weights") == 0) {457            if (arg_idx < argc-1) {458                included_weights.emplace_back(argv[++arg_idx]);459            } else {460                usage(argv[0]);461            }462        } else if (strcmp(argv[arg_idx], "--exclude-weights") == 0) {463            if (arg_idx < argc-1) {464                excluded_weights.emplace_back(argv[++arg_idx]);465            } else {466                usage(argv[0]);467            }468        } else if (strcmp(argv[arg_idx], "--keep-split") == 0) {469            params.keep_split = true;470        } else {471            usage(argv[0]);472        }473    }474 475    if (argc - arg_idx < 2) {476        printf("%s: bad arguments\n", argv[0]);477        usage(argv[0]);478    }479    if (!included_weights.empty() && !excluded_weights.empty()) {480        usage(argv[0]);481    }482 483    std::vector<std::string> imatrix_datasets;484    std::unordered_map<std::string, std::vector<float>> imatrix_data;485    int m_last_call = prepare_imatrix(imatrix_file, imatrix_datasets, included_weights, excluded_weights, imatrix_data);486 487    std::vector<llama_model_imatrix_data> i_data;488    std::vector<llama_model_tensor_override> t_override;489    if (!imatrix_data.empty()) {490        i_data.reserve(imatrix_data.size() + 1);491        for (const auto & kv : imatrix_data) {492            i_data.push_back({kv.first.c_str(), kv.second.data(), kv.second.size()});493        }494        i_data.push_back({nullptr, nullptr, 0});  // array terminator495        params.imatrix = i_data.data();496        {497            llama_model_kv_override kvo;498            std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_FILE);499            kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;500            strncpy(kvo.val_str, imatrix_file.c_str(), 127);501            kvo.val_str[127] = '\0';502            kv_overrides.emplace_back(std::move(kvo));503        }504        if (!imatrix_datasets.empty()) {505            llama_model_kv_override kvo;506            // TODO: list multiple datasets when there are more than one507            std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_DATASET);508            kvo.tag = LLAMA_KV_OVERRIDE_TYPE_STR;509            strncpy(kvo.val_str, imatrix_datasets[0].c_str(), 127);510            kvo.val_str[127] = '\0';511            kv_overrides.emplace_back(std::move(kvo));512        }513        {514            llama_model_kv_override kvo;515            std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_N_ENTRIES);516            kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;517            kvo.val_i64 = imatrix_data.size();518            kv_overrides.emplace_back(std::move(kvo));519        }520        if (m_last_call > 0) {521            llama_model_kv_override kvo;522            std::strcpy(kvo.key, LLM_KV_QUANTIZE_IMATRIX_N_CHUNKS);523            kvo.tag = LLAMA_KV_OVERRIDE_TYPE_INT;524            kvo.val_i64 = m_last_call;525            kv_overrides.emplace_back(std::move(kvo));526        }527    }528    if (!kv_overrides.empty()) {529        kv_overrides.emplace_back();530        kv_overrides.back().key[0] = 0;531        params.kv_overrides = kv_overrides.data();532    }533    if (!tensor_type_opts.empty()) {534        t_override.reserve(tensor_type_opts.size() + 1);535        for (const auto & tt : tensor_type_opts) {536            t_override.push_back({tt.name.c_str(), tt.type});537        }538        t_override.push_back({nullptr, GGML_TYPE_COUNT});  // array terminator539        params.tt_overrides = t_override.data();540    }541    if (!prune_layers.empty()) {542        prune_layers.push_back(-1);  // array terminator543        params.prune_layers = prune_layers.data();544    }545 546    llama_backend_init();547 548    // parse command line arguments549    const std::string fname_inp = argv[arg_idx];550    arg_idx++;551    std::string fname_out;552 553    std::string ftype_str;554    std::string suffix = ".gguf";555    if (try_parse_ftype(argv[arg_idx], params.ftype, ftype_str)) {556        // argv[arg_idx] is the ftype directly: <input> <ftype>557        if (!params.dry_run) {558            std::string fpath;559            const size_t pos = fname_inp.find_last_of("/\\");560            if (pos != std::string::npos) {561                fpath = fname_inp.substr(0, pos + 1);562            }563 564            // export as [inp path]/ggml-model-[ftype]. Only add extension if there is no splitting565            fname_out = fpath + "ggml-model-" + ftype_str;566            if (!params.keep_split) {567                fname_out += suffix;568            }569        }570        arg_idx++;571        if (ftype_str == "COPY") {572            params.only_copy = true;573        }574    } else {575        // argv[arg_idx] is not a valid ftype, so treat it as output path: <input> <output> <ftype>576        fname_out = argv[arg_idx];577        if (params.keep_split && fname_out.find(suffix) != std::string::npos) {578            fname_out = fname_out.substr(0, fname_out.length() - suffix.length());579        }580        arg_idx++;581 582        if (argc <= arg_idx) {583            fprintf(stderr, "%s: missing ftype\n", __func__);584            return 1;585        }586        if (!try_parse_ftype(argv[arg_idx], params.ftype, ftype_str)) {587            fprintf(stderr, "%s: invalid ftype '%s'\n", __func__, argv[arg_idx]);588            return 1;589        }590        if (ftype_str == "COPY") {591           params.only_copy = true;592        }593        arg_idx++;594    }595 596    // parse nthreads597    if (argc > arg_idx) {598        try {599            params.nthread = std::stoi(argv[arg_idx]);600        }601        catch (const std::exception & e) {602            fprintf(stderr, "%s: invalid nthread '%s' (%s)\n", __func__, argv[arg_idx], e.what());603            return 1;604        }605    }606 607    if (!params.dry_run) {608        if (std::error_code ec; std::filesystem::equivalent(fname_inp, fname_out, ec)) {609            fprintf(stderr, "%s: error: input and output files are the same: '%s'\n", __func__, fname_inp.c_str());610            return 1;611        }612    }613 614    llama_print_build_info();615 616    if (params.dry_run) {617        fprintf(stderr, "%s: calculating quantization size for '%s' as %s", __func__, fname_inp.c_str(), ftype_str.c_str());618    } else {619        fprintf(stderr, "%s: quantizing '%s' to '%s' as %s", __func__, fname_inp.c_str(), fname_out.c_str(), ftype_str.c_str());620    }621 622    if (params.nthread > 0) {623        fprintf(stderr, " using %d threads", params.nthread);624    }625    fprintf(stderr, "\n");626 627    const int64_t t_main_start_us = llama_time_us();628 629    int64_t t_quantize_us = 0;630 631    // load the model632    {633        const int64_t t_start_us = llama_time_us();634 635        if (llama_model_quantize(fname_inp.c_str(), fname_out.c_str(), &params)) {636            fprintf(stderr, "%s: failed to quantize model from '%s'\n", __func__, fname_inp.c_str());637            return 1;638        }639 640        t_quantize_us = llama_time_us() - t_start_us;641    }642 643    // report timing644    {645        const int64_t t_main_end_us = llama_time_us();646 647        printf("\n");648        printf("%s: quantize time = %8.2f ms\n", __func__, t_quantize_us/1000.0);649        printf("%s:    total time = %8.2f ms\n", __func__, (t_main_end_us - t_main_start_us)/1000.0);650    }651 652    llama_backend_free();653 654    return 0;655}656