Xenobd/whisper.cpp
0
1#include "common-ggml.h"2 3#include <regex>4#include <map>5 6static const std::map<std::string, enum ggml_ftype> GGML_FTYPE_MAP = {7 {"q4_0", GGML_FTYPE_MOSTLY_Q4_0},8 {"q4_1", GGML_FTYPE_MOSTLY_Q4_1},9 {"q5_0", GGML_FTYPE_MOSTLY_Q5_0},10 {"q5_1", GGML_FTYPE_MOSTLY_Q5_1},11 {"q8_0", GGML_FTYPE_MOSTLY_Q8_0},12 {"q2_k", GGML_FTYPE_MOSTLY_Q2_K},13 {"q3_k", GGML_FTYPE_MOSTLY_Q3_K},14 {"q4_k", GGML_FTYPE_MOSTLY_Q4_K},15 {"q5_k", GGML_FTYPE_MOSTLY_Q5_K},16 {"q6_k", GGML_FTYPE_MOSTLY_Q6_K},17};18 19void ggml_print_ftypes(FILE * fp) {20 for (auto it = GGML_FTYPE_MAP.begin(); it != GGML_FTYPE_MAP.end(); it++) {21 fprintf(fp, " type = \"%s\" or %d\n", it->first.c_str(), it->second);22 }23}24 25enum ggml_ftype ggml_parse_ftype(const char * str) {26 enum ggml_ftype ftype;27 if (str[0] == 'q') {28 const auto it = GGML_FTYPE_MAP.find(str);29 if (it == GGML_FTYPE_MAP.end()) {30 fprintf(stderr, "%s: unknown ftype '%s'\n", __func__, str);31 return GGML_FTYPE_UNKNOWN;32 }33 ftype = it->second;34 } else {35 ftype = (enum ggml_ftype) atoi(str);36 }37 38 return ftype;39}40 41bool ggml_common_quantize_0(42 std::ifstream & finp,43 std::ofstream & fout,44 const ggml_ftype ftype,45 const std::vector<std::string> & to_quant,46 const std::vector<std::string> & to_skip) {47 48 ggml_type qtype = GGML_TYPE_F32;49 50 switch (ftype) {51 case GGML_FTYPE_MOSTLY_Q4_0: qtype = GGML_TYPE_Q4_0; break;52 case GGML_FTYPE_MOSTLY_Q4_1: qtype = GGML_TYPE_Q4_1; break;53 case GGML_FTYPE_MOSTLY_Q5_0: qtype = GGML_TYPE_Q5_0; break;54 case GGML_FTYPE_MOSTLY_Q5_1: qtype = GGML_TYPE_Q5_1; break;55 case GGML_FTYPE_MOSTLY_Q8_0: qtype = GGML_TYPE_Q8_0; break;56 case GGML_FTYPE_MOSTLY_Q2_K: qtype = GGML_TYPE_Q2_K; break;57 case GGML_FTYPE_MOSTLY_Q3_K: qtype = GGML_TYPE_Q3_K; break;58 case GGML_FTYPE_MOSTLY_Q4_K: qtype = GGML_TYPE_Q4_K; break;59 case GGML_FTYPE_MOSTLY_Q5_K: qtype = GGML_TYPE_Q5_K; break;60 case GGML_FTYPE_MOSTLY_Q6_K: qtype = GGML_TYPE_Q6_K; break;61 case GGML_FTYPE_UNKNOWN:62 case GGML_FTYPE_ALL_F32:63 case GGML_FTYPE_MOSTLY_F16:64 case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16:65 case GGML_FTYPE_MOSTLY_IQ2_XXS:66 case GGML_FTYPE_MOSTLY_IQ2_XS:67 case GGML_FTYPE_MOSTLY_IQ2_S:68 case GGML_FTYPE_MOSTLY_IQ3_XXS:69 case GGML_FTYPE_MOSTLY_IQ3_S:70 case GGML_FTYPE_MOSTLY_IQ1_S:71 case GGML_FTYPE_MOSTLY_IQ4_NL:72 case GGML_FTYPE_MOSTLY_IQ4_XS:73 case GGML_FTYPE_MOSTLY_IQ1_M:74 case GGML_FTYPE_MOSTLY_BF16:75 case GGML_FTYPE_MOSTLY_MXFP4:76 {77 fprintf(stderr, "%s: invalid model type %d\n", __func__, ftype);78 return false;79 }80 };81 82 if (!ggml_is_quantized(qtype)) {83 fprintf(stderr, "%s: invalid quantization type %d (%s)\n", __func__, qtype, ggml_type_name(qtype));84 return false;85 }86 87 size_t total_size_org = 0;88 size_t total_size_new = 0;89 90 std::vector<float> work;91 92 std::vector<uint8_t> data_u8;93 std::vector<ggml_fp16_t> data_f16;94 std::vector<float> data_f32;95 96 while (true) {97 int32_t n_dims;98 int32_t length;99 int32_t ttype;100 101 finp.read(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));102 finp.read(reinterpret_cast<char *>(&length), sizeof(length));103 finp.read(reinterpret_cast<char *>(&ttype), sizeof(ttype));104 105 if (finp.eof()) {106 break;107 }108 109 int32_t nelements = 1;110 int32_t ne[4] = { 1, 1, 1, 1 };111 for (int i = 0; i < n_dims; ++i) {112 finp.read (reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));113 nelements *= ne[i];114 }115 116 std::string name(length, 0);117 finp.read (&name[0], length);118 119 printf("%64s - [%5d, %5d, %5d], type = %6s ", name.data(), ne[0], ne[1], ne[2], ggml_type_name((ggml_type) ttype));120 121 bool quantize = false;122 123 // check if we should quantize this tensor124 for (const auto & s : to_quant) {125 if (std::regex_match(name, std::regex(s))) {126 quantize = true;127 break;128 }129 }130 131 // check if we should skip this tensor132 for (const auto & s : to_skip) {133 if (std::regex_match(name, std::regex(s))) {134 quantize = false;135 break;136 }137 }138 139 // quantize only 2D tensors140 quantize &= (n_dims == 2);141 142 if (quantize) {143 if (ttype != GGML_TYPE_F32 && ttype != GGML_TYPE_F16) {144 fprintf(stderr, "%s: unsupported ttype %d (%s) for integer quantization\n", __func__, ttype, ggml_type_name((ggml_type) ttype));145 return false;146 }147 148 if (ttype == GGML_TYPE_F16) {149 data_f16.resize(nelements);150 finp.read(reinterpret_cast<char *>(data_f16.data()), nelements * sizeof(ggml_fp16_t));151 data_f32.resize(nelements);152 for (int i = 0; i < nelements; ++i) {153 data_f32[i] = ggml_fp16_to_fp32(data_f16[i]);154 }155 } else {156 data_f32.resize(nelements);157 finp.read(reinterpret_cast<char *>(data_f32.data()), nelements * sizeof(float));158 }159 160 ttype = qtype;161 } else {162 const int bpe = (ttype == 0) ? sizeof(float) : sizeof(uint16_t);163 164 data_u8.resize(nelements*bpe);165 finp.read(reinterpret_cast<char *>(data_u8.data()), nelements * bpe);166 }167 168 fout.write(reinterpret_cast<char *>(&n_dims), sizeof(n_dims));169 fout.write(reinterpret_cast<char *>(&length), sizeof(length));170 fout.write(reinterpret_cast<char *>(&ttype), sizeof(ttype));171 for (int i = 0; i < n_dims; ++i) {172 fout.write(reinterpret_cast<char *>(&ne[i]), sizeof(ne[i]));173 }174 fout.write(&name[0], length);175 176 if (quantize) {177 work.resize(nelements); // for quantization178 179 size_t cur_size = 0;180 switch ((ggml_type) ttype) {181 case GGML_TYPE_Q4_0:182 case GGML_TYPE_Q4_1:183 case GGML_TYPE_Q5_0:184 case GGML_TYPE_Q5_1:185 case GGML_TYPE_Q8_0:186 case GGML_TYPE_Q2_K:187 case GGML_TYPE_Q3_K:188 case GGML_TYPE_Q4_K:189 case GGML_TYPE_Q5_K:190 case GGML_TYPE_Q6_K:191 {192 cur_size = ggml_quantize_chunk((ggml_type) ttype, data_f32.data(), work.data(), 0, nelements/ne[0], ne[0], nullptr);193 } break;194 case GGML_TYPE_F32:195 case GGML_TYPE_F16:196 case GGML_TYPE_I8:197 case GGML_TYPE_I16:198 case GGML_TYPE_I32:199 case GGML_TYPE_I64:200 case GGML_TYPE_F64:201 case GGML_TYPE_Q8_1:202 case GGML_TYPE_Q8_K:203 case GGML_TYPE_IQ2_XXS:204 case GGML_TYPE_IQ2_XS:205 case GGML_TYPE_IQ2_S:206 case GGML_TYPE_IQ3_XXS:207 case GGML_TYPE_IQ3_S:208 case GGML_TYPE_IQ1_S:209 case GGML_TYPE_IQ4_NL:210 case GGML_TYPE_IQ4_XS:211 case GGML_TYPE_IQ1_M:212 case GGML_TYPE_BF16:213 case GGML_TYPE_TQ1_0:214 case GGML_TYPE_TQ2_0:215 case GGML_TYPE_MXFP4:216 case GGML_TYPE_COUNT:217 {218 fprintf(stderr, "%s: unsupported quantization type %d (%s)\n", __func__, ttype, ggml_type_name((ggml_type) ttype));219 return false;220 }221 }222 223 fout.write(reinterpret_cast<char *>(work.data()), cur_size);224 total_size_new += cur_size;225 226 printf("size = %8.2f MB -> %8.2f MB\n", nelements * sizeof(float)/1024.0/1024.0, cur_size/1024.0/1024.0);227 } else {228 printf("size = %8.3f MB\n", data_u8.size()/1024.0/1024.0);229 fout.write(reinterpret_cast<char *>(data_u8.data()), data_u8.size());230 total_size_new += data_u8.size();231 }232 233 total_size_org += nelements * sizeof(float);234 }235 236 printf("%s: model size = %8.2f MB\n", __func__, total_size_org/1024.0/1024.0);237 printf("%s: quant size = %8.2f MB | ftype = %d (%s)\n", __func__, total_size_new/1024.0/1024.0, ftype, ggml_type_name(qtype));238 239 return true;240}241