KBaba7/llama.cpp
0
1#include "ggml.h"2#include "ggml-backend.h"3#include "ggml-impl.h"4#include "gguf.h"5 6#include <cinttypes>7#include <cstddef>8#include <cstdint>9#include <cstdio>10#include <cstdlib>11#include <cstring>12#include <map>13#include <new>14#include <stdexcept>15#include <string>16#include <vector>17 18template <typename T>19struct type_to_gguf_type;20 21template <>22struct type_to_gguf_type<uint8_t> {23 static constexpr enum gguf_type value = GGUF_TYPE_UINT8;24};25 26template <>27struct type_to_gguf_type<int8_t> {28 static constexpr enum gguf_type value = GGUF_TYPE_INT8;29};30 31template <>32struct type_to_gguf_type<uint16_t> {33 static constexpr enum gguf_type value = GGUF_TYPE_UINT16;34};35 36template <>37struct type_to_gguf_type<int16_t> {38 static constexpr enum gguf_type value = GGUF_TYPE_INT16;39};40 41template <>42struct type_to_gguf_type<uint32_t> {43 static constexpr enum gguf_type value = GGUF_TYPE_UINT32;44};45 46template <>47struct type_to_gguf_type<int32_t> {48 static constexpr enum gguf_type value = GGUF_TYPE_INT32;49};50 51template <>52struct type_to_gguf_type<float> {53 static constexpr enum gguf_type value = GGUF_TYPE_FLOAT32;54};55 56template <>57struct type_to_gguf_type<bool> {58 static constexpr enum gguf_type value = GGUF_TYPE_BOOL;59};60 61template <>62struct type_to_gguf_type<std::string> {63 static constexpr enum gguf_type value = GGUF_TYPE_STRING;64};65 66template <>67struct type_to_gguf_type<uint64_t> {68 static constexpr enum gguf_type value = GGUF_TYPE_UINT64;69};70 71template <>72struct type_to_gguf_type<int64_t> {73 static constexpr enum gguf_type value = GGUF_TYPE_INT64;74};75 76template <>77struct type_to_gguf_type<double> {78 static constexpr enum gguf_type value = GGUF_TYPE_FLOAT64;79};80 81static const std::map<gguf_type, size_t> GGUF_TYPE_SIZE = {82 {GGUF_TYPE_UINT8, sizeof(uint8_t)},83 {GGUF_TYPE_INT8, sizeof(int8_t)},84 {GGUF_TYPE_UINT16, sizeof(uint16_t)},85 {GGUF_TYPE_INT16, sizeof(int16_t)},86 {GGUF_TYPE_UINT32, sizeof(uint32_t)},87 {GGUF_TYPE_INT32, sizeof(int32_t)},88 {GGUF_TYPE_FLOAT32, sizeof(float)},89 {GGUF_TYPE_BOOL, sizeof(int8_t)},90 {GGUF_TYPE_STRING, 0}, // undefined91 {GGUF_TYPE_ARRAY, 0}, // undefined92 {GGUF_TYPE_UINT64, sizeof(uint64_t)},93 {GGUF_TYPE_INT64, sizeof(int64_t)},94 {GGUF_TYPE_FLOAT64, sizeof(double)},95};96static_assert(GGUF_TYPE_COUNT == 13, "GGUF_TYPE_COUNT != 13");97 98static const std::map<gguf_type, const char *> GGUF_TYPE_NAME = {99 {GGUF_TYPE_UINT8, "u8"},100 {GGUF_TYPE_INT8, "i8"},101 {GGUF_TYPE_UINT16, "u16"},102 {GGUF_TYPE_INT16, "i16"},103 {GGUF_TYPE_UINT32, "u32"},104 {GGUF_TYPE_INT32, "i32"},105 {GGUF_TYPE_FLOAT32, "f32"},106 {GGUF_TYPE_BOOL, "bool"},107 {GGUF_TYPE_STRING, "str"},108 {GGUF_TYPE_ARRAY, "arr"},109 {GGUF_TYPE_UINT64, "u64"},110 {GGUF_TYPE_INT64, "i64"},111 {GGUF_TYPE_FLOAT64, "f64"},112};113static_assert(GGUF_TYPE_COUNT == 13, "GGUF_TYPE_COUNT != 13");114 115size_t gguf_type_size(enum gguf_type type) {116 auto it = GGUF_TYPE_SIZE.find(type);117 return it == GGUF_TYPE_SIZE.end() ? 0 : it->second;118}119 120struct gguf_kv {121 std::string key;122 123 bool is_array;124 enum gguf_type type;125 126 std::vector<int8_t> data;127 std::vector<std::string> data_string;128 129 template <typename T>130 gguf_kv(const std::string & key, const T value)131 : key(key), is_array(false), type(type_to_gguf_type<T>::value) {132 GGML_ASSERT(!key.empty());133 data.resize(sizeof(T));134 memcpy(data.data(), &value, sizeof(T));135 }136 137 template <typename T>138 gguf_kv(const std::string & key, const std::vector<T> & value)139 : key(key), is_array(true), type(type_to_gguf_type<T>::value) {140 GGML_ASSERT(!key.empty());141 data.resize(value.size()*sizeof(T));142 for (size_t i = 0; i < value.size(); ++i) {143 const T tmp = value[i];144 memcpy(data.data() + i*sizeof(T), &tmp, sizeof(T));145 }146 }147 148 gguf_kv(const std::string & key, const std::string & value)149 : key(key), is_array(false), type(GGUF_TYPE_STRING) {150 GGML_ASSERT(!key.empty());151 data_string.push_back(value);152 }153 154 gguf_kv(const std::string & key, const std::vector<std::string> & value)155 : key(key), is_array(true), type(GGUF_TYPE_STRING) {156 GGML_ASSERT(!key.empty());157 data_string = value;158 }159 160 const std::string & get_key() const {161 return key;162 }163 164 const enum gguf_type & get_type() const {165 return type;166 }167 168 size_t get_ne() const {169 if (type == GGUF_TYPE_STRING) {170 const size_t ne = data_string.size();171 GGML_ASSERT(is_array || ne == 1);172 return ne;173 }174 const size_t type_size = gguf_type_size(type);175 GGML_ASSERT(data.size() % type_size == 0);176 const size_t ne = data.size() / type_size;177 GGML_ASSERT(is_array || ne == 1);178 return ne;179 }180 181 template <typename T>182 const T & get_val(const size_t i = 0) const {183 GGML_ASSERT(type_to_gguf_type<T>::value == type);184 if constexpr (std::is_same<T, std::string>::value) {185 GGML_ASSERT(data_string.size() >= i+1);186 return data_string[i];187 }188 const size_t type_size = gguf_type_size(type);189 GGML_ASSERT(data.size() % type_size == 0);190 GGML_ASSERT(data.size() >= (i+1)*type_size);191 return reinterpret_cast<const T *>(data.data())[i];192 }193 194 void cast(const enum gguf_type new_type) {195 const size_t new_type_size = gguf_type_size(new_type);196 GGML_ASSERT(data.size() % new_type_size == 0);197 type = new_type;198 }199};200 201struct gguf_tensor_info {202 struct ggml_tensor t; // for holding the equivalent info203 uint64_t offset; // offset from start of `data`, must be a multiple of `ALIGNMENT`204};205 206struct gguf_context {207 uint32_t version = GGUF_VERSION;208 209 std::vector<struct gguf_kv> kv;210 std::vector<struct gguf_tensor_info> info;211 212 size_t alignment = GGUF_DEFAULT_ALIGNMENT;213 size_t offset = 0; // offset of `data` from beginning of file214 size_t size = 0; // size of `data` in bytes215 216 void * data = nullptr;217};218 219struct gguf_reader {220 FILE * file;221 222 gguf_reader(FILE * file) : file(file) {}223 224 template <typename T>225 bool read(T & dst) const {226 return fread(&dst, 1, sizeof(dst), file) == sizeof(dst);227 }228 229 template <typename T>230 bool read(std::vector<T> & dst, const size_t n) const {231 dst.resize(n);232 for (size_t i = 0; i < dst.size(); ++i) {233 if constexpr (std::is_same<T, bool>::value) {234 bool tmp;235 if (!read(tmp)) {236 return false;237 }238 dst[i] = tmp;239 } else {240 if (!read(dst[i])) {241 return false;242 }243 }244 }245 return true;246 }247 248 bool read(bool & dst) const {249 int8_t tmp = -1;250 if (!read(tmp)) {251 return false;252 }253 dst = tmp != 0;254 return true;255 }256 257 bool read(enum ggml_type & dst) const {258 int32_t tmp = -1;259 if (!read(tmp)) {260 return false;261 }262 dst = ggml_type(tmp);263 return true;264 }265 266 bool read(enum gguf_type & dst) const {267 int32_t tmp = -1;268 if (!read(tmp)) {269 return false;270 }271 dst = gguf_type(tmp);272 return true;273 }274 275 bool read(std::string & dst) const {276 uint64_t size = -1;277 if (!read(size)) {278 return false;279 }280 dst.resize(size);281 return fread(dst.data(), 1, dst.length(), file) == dst.length();282 }283 284 bool read(void * dst, const size_t size) const {285 return fread(dst, 1, size, file) == size;286 }287};288 289struct gguf_context * gguf_init_empty(void) {290 return new gguf_context;291}292 293template<typename T>294bool gguf_read_emplace_helper(const struct gguf_reader & gr, std::vector<struct gguf_kv> & kv, const std::string & key, const bool is_array, const size_t n) {295 if (is_array) {296 std::vector<T> value;297 try {298 if (!gr.read(value, n)) {299 return false;300 }301 } catch (std::length_error &) {302 fprintf(stderr, "%s: encountered length_error while reading value for key '%s'\n", __func__, key.c_str());303 return false;304 } catch (std::bad_alloc &) {305 fprintf(stderr, "%s: encountered bad_alloc error while reading value for key '%s'\n", __func__, key.c_str());306 return false;307 }308 kv.emplace_back(key, value);309 } else {310 T value;311 if (!gr.read(value)) {312 return false;313 }314 kv.emplace_back(key, value);315 }316 return true;317}318 319struct gguf_context * gguf_init_from_file_impl(FILE * file, struct gguf_init_params params) {320 const struct gguf_reader gr(file);321 struct gguf_context * ctx = new gguf_context;322 323 bool ok = true;324 325 // file magic326 {327 std::vector<char> magic;328 ok = ok && gr.read(magic, 4);329 330 if (!ok) {331 fprintf(stderr, "%s: failed to read magic\n", __func__);332 gguf_free(ctx);333 return nullptr;334 }335 336 for (uint32_t i = 0; i < magic.size(); i++) {337 if (magic[i] != GGUF_MAGIC[i]) {338 fprintf(stderr, "%s: invalid magic characters: '%c%c%c%c', expected 'GGUF'\n", __func__, magic[0], magic[1], magic[2], magic[3]);339 gguf_free(ctx);340 return nullptr;341 }342 }343 }344 345 // header346 int64_t n_kv = 0;347 int64_t n_tensors = 0;348 349 if (ok && gr.read(ctx->version)) {350 if (ctx->version == 1) {351 fprintf(stderr, "%s: GGUFv1 is no longer supported, please use a more up-to-date version\n", __func__);352 ok = false;353 }354 if (ctx->version > GGUF_VERSION) {355 fprintf(stderr, "%s: this GGUF file is version %" PRIu32 " but this software only supports up to version %d\n",356 __func__, ctx->version, GGUF_VERSION);357 ok = false;358 }359 } else {360 ok = false;361 }362 363 if (ok && gr.read(n_tensors)) {364 static_assert(sizeof(size_t) <= 8 && sizeof(gguf_tensor_info) >= 2, "int64_t insufficient for indexing");365 if (n_tensors < 0 || n_tensors > int64_t(SIZE_MAX/sizeof(gguf_tensor_info))) {366 fprintf(stderr, "%s: number of tensors is %" PRIi64 " but must be in [0, %zu]\n",367 __func__, n_tensors, SIZE_MAX/sizeof(gguf_tensor_info));368 ok = false;369 }370 } else {371 ok = false;372 }373 374 if (ok && gr.read(n_kv)) {375 static_assert(sizeof(size_t) <= 8 && sizeof(gguf_tensor_info) >= 2, "int64_t insufficient for indexing");376 if (n_kv < 0 || n_kv > int64_t(SIZE_MAX/sizeof(gguf_kv))) {377 fprintf(stderr, "%s: number of key value pairs is %" PRIi64 " but must be in [0, %zu]\n",378 __func__, n_kv, SIZE_MAX/sizeof(gguf_kv));379 ok = false;380 }381 } else {382 ok = false;383 }384 385 if (!ok) {386 fprintf(stderr, "%s: failed to read header\n", __func__);387 gguf_free(ctx);388 return nullptr;389 }390 391 // KV pairs392 {393 for (int64_t i = 0; ok && i < n_kv; ++i) {394 std::string key;395 gguf_type type = gguf_type(-1);396 bool is_array = false;397 uint64_t n = 1;398 399 try {400 ok = ok && gr.read(key);401 } catch (std::length_error &) {402 fprintf(stderr, "%s: encountered length_error while reading key %" PRIi64 "\n", __func__, i);403 ok = false;404 } catch (std::bad_alloc &) {405 fprintf(stderr, "%s: encountered bad_alloc error while reading key %" PRIi64 "\n", __func__, i);406 ok = false;407 }408 for (size_t j = 0; ok && j < ctx->kv.size(); ++j) {409 if (key == ctx->kv[j].key) {410 fprintf(stderr, "%s: duplicate key '%s' for tensors %zu and %" PRIi64 " \n", __func__, key.c_str(), j, i);411 ok = false;412 }413 }414 if (!ok) {415 break;416 }417 418 ok = ok && gr.read(type);419 if (type == GGUF_TYPE_ARRAY) {420 is_array = true;421 ok = ok && gr.read(type);422 ok = ok && gr.read(n);423 }424 if (!ok) {425 break;426 }427 428 switch (type) {429 case GGUF_TYPE_UINT8: ok = ok && gguf_read_emplace_helper<uint8_t> (gr, ctx->kv, key, is_array, n); break;430 case GGUF_TYPE_INT8: ok = ok && gguf_read_emplace_helper<int8_t> (gr, ctx->kv, key, is_array, n); break;431 case GGUF_TYPE_UINT16: ok = ok && gguf_read_emplace_helper<uint16_t> (gr, ctx->kv, key, is_array, n); break;432 case GGUF_TYPE_INT16: ok = ok && gguf_read_emplace_helper<int16_t> (gr, ctx->kv, key, is_array, n); break;433 case GGUF_TYPE_UINT32: ok = ok && gguf_read_emplace_helper<uint32_t> (gr, ctx->kv, key, is_array, n); break;434 case GGUF_TYPE_INT32: ok = ok && gguf_read_emplace_helper<int32_t> (gr, ctx->kv, key, is_array, n); break;435 case GGUF_TYPE_FLOAT32: ok = ok && gguf_read_emplace_helper<float> (gr, ctx->kv, key, is_array, n); break;436 case GGUF_TYPE_BOOL: ok = ok && gguf_read_emplace_helper<bool> (gr, ctx->kv, key, is_array, n); break;437 case GGUF_TYPE_STRING: ok = ok && gguf_read_emplace_helper<std::string>(gr, ctx->kv, key, is_array, n); break;438 case GGUF_TYPE_UINT64: ok = ok && gguf_read_emplace_helper<uint64_t> (gr, ctx->kv, key, is_array, n); break;439 case GGUF_TYPE_INT64: ok = ok && gguf_read_emplace_helper<int64_t> (gr, ctx->kv, key, is_array, n); break;440 case GGUF_TYPE_FLOAT64: ok = ok && gguf_read_emplace_helper<double> (gr, ctx->kv, key, is_array, n); break;441 case GGUF_TYPE_ARRAY:442 default:443 {444 fprintf(stderr, "%s: key '%s' has invalid GGUF type %d\n", __func__, key.c_str(), type);445 ok = false;446 } break;447 }448 }449 450 if (!ok) {451 fprintf(stderr, "%s: failed to read key-value pairs\n", __func__);452 gguf_free(ctx);453 return nullptr;454 }455 GGML_ASSERT(int64_t(ctx->kv.size()) == n_kv);456 457 const int alignment_idx = gguf_find_key(ctx, GGUF_KEY_GENERAL_ALIGNMENT);458 ctx->alignment = alignment_idx == -1 ? GGUF_DEFAULT_ALIGNMENT : gguf_get_val_u32(ctx, alignment_idx);459 460 if (ctx->alignment == 0 || (ctx->alignment & (ctx->alignment - 1)) != 0) {461 fprintf(stderr, "%s: alignment %zu is not a power of 2\n", __func__, ctx->alignment);462 gguf_free(ctx);463 return nullptr;464 }465 }466 467 // read the tensor info468 for (int64_t i = 0; ok && i < n_tensors; ++i) {469 struct gguf_tensor_info info;470 471 // tensor name472 {473 std::string name;474 try {475 ok = ok && gr.read(name);476 } catch (std::length_error &) {477 fprintf(stderr, "%s: encountered length_error while reading tensor name %" PRIi64 "\n", __func__, i);478 ok = false;479 } catch (std::bad_alloc &) {480 fprintf(stderr, "%s: encountered bad_alloc error while reading tensor name %" PRIi64 "\n", __func__, i);481 ok = false;482 }483 if (name.length() >= GGML_MAX_NAME) {484 fprintf(stderr, "%s: tensor name %" PRIi64 " is too long: %zu >= %d\n", __func__, i, name.length(), GGML_MAX_NAME);485 ok = false;486 break;487 }488 ggml_set_name(&info.t, name.c_str());489 490 // make sure there are no duplicate tensor names491 for (int64_t j = 0; ok && j < i; ++j) {492 if (strcmp(info.t.name, ctx->info[j].t.name) == 0) {493 fprintf(stderr, "%s: duplicate tensor name '%s' for tensors %" PRIi64 " and %" PRIi64 "\n", __func__, info.t.name, j, i);494 ok = false;495 break;496 }497 }498 }499 if (!ok) {500 break;501 }502 503 // tensor shape504 {505 uint32_t n_dims = -1;506 ok = ok && gr.read(n_dims);507 if (n_dims > GGML_MAX_DIMS) {508 fprintf(stderr, "%s: tensor '%s' has invalid number of dimensions: %" PRIu32 " > %" PRIu32 "\n",509 __func__, info.t.name, n_dims, GGML_MAX_DIMS);510 ok = false;511 break;512 }513 for (uint32_t j = 0; ok && j < GGML_MAX_DIMS; ++j) {514 info.t.ne[j] = 1;515 if (j < n_dims) {516 ok = ok && gr.read(info.t.ne[j]);517 }518 519 // check that all ne are non-negative520 if (info.t.ne[j] < 0) {521 fprintf(stderr, "%s: tensor '%s' dimension %" PRIu32 " has invalid number of elements: %" PRIi64 " < 0\n",522 __func__, info.t.name, j, info.t.ne[j]);523 ok = false;524 break;525 }526 }527 528 // check that the total number of elements is representable529 if (ok && ((INT64_MAX/info.t.ne[1] <= info.t.ne[0]) ||530 (INT64_MAX/info.t.ne[2] <= info.t.ne[0]*info.t.ne[1]) ||531 (INT64_MAX/info.t.ne[3] <= info.t.ne[0]*info.t.ne[1]*info.t.ne[2]))) {532 533 fprintf(stderr, "%s: total number of elements in tensor '%s' with shape "534 "(%" PRIi64 ", %" PRIi64 ", %" PRIi64 ", %" PRIi64 ") is >= %" PRIi64 "\n",535 __func__, info.t.name, info.t.ne[0], info.t.ne[1], info.t.ne[2], info.t.ne[3], INT64_MAX);536 ok = false;537 break;538 }539 }540 if (!ok) {541 break;542 }543 544 // tensor type545 {546 ok = ok && gr.read(info.t.type);547 548 // check that tensor type is within defined range549 if (info.t.type < 0 || info.t.type >= GGML_TYPE_COUNT) {550 fprintf(stderr, "%s: tensor '%s' has invalid ggml type %d (%s)\n",551 __func__, info.t.name, info.t.type, ggml_type_name(info.t.type));552 ok = false;553 break;554 }555 const size_t type_size = ggml_type_size(info.t.type);556 const int64_t blck_size = ggml_blck_size(info.t.type);557 558 // check that row size is divisible by block size559 if (blck_size == 0 || info.t.ne[0] % blck_size != 0) {560 fprintf(stderr, "%s: tensor '%s' of type %d (%s) has %" PRId64 " elements per row, "561 "not a multiple of block size (%" PRId64 ")\n",562 __func__, info.t.name, (int) info.t.type, ggml_type_name(info.t.type), info.t.ne[0], blck_size);563 ok = false;564 break;565 }566 567 // calculate byte offsets given the tensor shape and type568 info.t.nb[0] = type_size;569 info.t.nb[1] = info.t.nb[0]*(info.t.ne[0]/blck_size);570 for (int j = 2; j < GGML_MAX_DIMS; ++j) {571 info.t.nb[j] = info.t.nb[j - 1]*info.t.ne[j - 1];572 }573 }574 if (!ok) {575 break;576 }577 578 // tensor data offset within buffer579 ok = ok && gr.read(info.offset);580 581 ctx->info.push_back(info);582 }583 584 if (!ok) {585 fprintf(stderr, "%s: failed to read tensor info\n", __func__);586 gguf_free(ctx);587 return nullptr;588 }589 GGML_ASSERT(int64_t(ctx->info.size()) == n_tensors);590 591 // we require the data section to be aligned, so take into account any padding592 if (fseek(file, GGML_PAD(ftell(file), ctx->alignment), SEEK_SET) != 0) {593 fprintf(stderr, "%s: failed to seek to beginning of data section\n", __func__);594 gguf_free(ctx);595 return nullptr;596 }597 598 // store the current file offset - this is where the data section starts599 ctx->offset = ftell(file);600 601 // compute the total size of the data section, taking into account the alignment602 {603 ctx->size = 0;604 for (size_t i = 0; i < ctx->info.size(); ++i) {605 const gguf_tensor_info & ti = ctx->info[i];606 if (ti.offset != ctx->size) {607 fprintf(stderr, "%s: tensor '%s' has offset %" PRIu64 ", expected %zu\n",608 __func__, ti.t.name, ti.offset, ctx->size);609 fprintf(stderr, "%s: failed to read tensor data\n", __func__);610 gguf_free(ctx);611 return nullptr;612 }613 ctx->size += GGML_PAD(ggml_nbytes(&ti.t), ctx->alignment);614 }615 }616 617 // load the tensor data only if requested618 if (params.ctx != nullptr) {619 // if the provided gguf_context is no_alloc, then we create "empty" tensors and do not read the binary blob620 // otherwise, we load the binary blob into the created ggml_context as well, and point the "data" members of621 // the ggml_tensor structs to the appropriate locations in the binary blob622 623 // compute the exact size needed for the new ggml_context624 const size_t mem_size =625 params.no_alloc ?626 (n_tensors )*ggml_tensor_overhead() :627 (n_tensors + 1)*ggml_tensor_overhead() + ctx->size;628 629 struct ggml_init_params pdata = {630 /*mem_size =*/ mem_size,631 /*mem_buffer =*/ nullptr,632 /*no_alloc =*/ params.no_alloc,633 };634 635 *params.ctx = ggml_init(pdata);636 if (*params.ctx == nullptr) {637 fprintf(stderr, "%s: failed to initialize ggml context for storing tensors\n", __func__);638 gguf_free(ctx);639 return nullptr;640 }641 642 struct ggml_context * ctx_data = *params.ctx;643 644 struct ggml_tensor * data = nullptr;645 646 if (!params.no_alloc) {647 data = ggml_new_tensor_1d(ctx_data, GGML_TYPE_I8, ctx->size);648 649 ok = ok && data != nullptr;650 651 if (ok) {652 ggml_set_name(data, "GGUF tensor data binary blob");653 }654 655 // read the binary blob with the tensor data656 ok = ok && gr.read(data->data, ctx->size);657 658 if (!ok) {659 fprintf(stderr, "%s: failed to read tensor data binary blob\n", __func__);660 ggml_free(ctx_data);661 *params.ctx = nullptr;662 gguf_free(ctx);663 return nullptr;664 }665 666 ctx->data = data->data;667 }668 669 ggml_set_no_alloc(ctx_data, true);670 671 // create the tensors672 for (size_t i = 0; i < ctx->info.size(); ++i) {673 const struct gguf_tensor_info & info = ctx->info[i];674 675 struct ggml_tensor * cur = ggml_new_tensor(ctx_data, info.t.type, GGML_MAX_DIMS, info.t.ne);676 677 ok = ok && cur != nullptr;678 679 if (!ok) {680 break;681 }682 683 ggml_set_name(cur, info.t.name);684 685 // point the data member to the appropriate location in the binary blob using the tensor info686 if (!params.no_alloc) {687 cur->data = (char *) data->data + info.offset;688 }689 }690 691 if (!ok) {692 fprintf(stderr, "%s: failed to create tensors\n", __func__);693 ggml_free(ctx_data);694 *params.ctx = nullptr;695 gguf_free(ctx);696 return nullptr;697 }698 699 ggml_set_no_alloc(ctx_data, params.no_alloc);700 }701 702 return ctx;703}704 705struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params) {706 FILE * file = ggml_fopen(fname, "rb");707 708 if (!file) {709 fprintf(stderr, "%s: failed to open GGUF file '%s'\n", __func__, fname);710 return nullptr;711 }712 713 struct gguf_context * result = gguf_init_from_file_impl(file, params);714 fclose(file);715 return result;716}717 718void gguf_free(struct gguf_context * ctx) {719 if (ctx == nullptr) {720 return;721 }722 delete ctx;723}724 725const char * gguf_type_name(enum gguf_type type) {726 auto it = GGUF_TYPE_NAME.find(type);727 return it == GGUF_TYPE_NAME.end() ? nullptr : it->second;728}729 730uint32_t gguf_get_version(const struct gguf_context * ctx) {731 return ctx->version;732}733 734size_t gguf_get_alignment(const struct gguf_context * ctx) {735 return ctx->alignment;736}737 738size_t gguf_get_data_offset(const struct gguf_context * ctx) {739 return ctx->offset;740}741 742int64_t gguf_get_n_kv(const struct gguf_context * ctx) {743 return ctx->kv.size();744}745 746int64_t gguf_find_key(const struct gguf_context * ctx, const char * key) {747 // return -1 if key not found748 int64_t keyfound = -1;749 750 const int64_t n_kv = gguf_get_n_kv(ctx);751 752 for (int64_t i = 0; i < n_kv; ++i) {753 if (strcmp(key, gguf_get_key(ctx, i)) == 0) {754 keyfound = i;755 break;756 }757 }758 759 return keyfound;760}761 762const char * gguf_get_key(const struct gguf_context * ctx, int64_t key_id) {763 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));764 return ctx->kv[key_id].get_key().c_str();765}766 767enum gguf_type gguf_get_kv_type(const struct gguf_context * ctx, int64_t key_id) {768 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));769 return ctx->kv[key_id].is_array ? GGUF_TYPE_ARRAY : ctx->kv[key_id].get_type();770}771 772enum gguf_type gguf_get_arr_type(const struct gguf_context * ctx, int64_t key_id) {773 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));774 GGML_ASSERT(ctx->kv[key_id].is_array);775 return ctx->kv[key_id].get_type();776}777 778const void * gguf_get_arr_data(const struct gguf_context * ctx, int64_t key_id) {779 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));780 GGML_ASSERT(ctx->kv[key_id].get_type() != GGUF_TYPE_STRING);781 return ctx->kv[key_id].data.data();782}783 784const char * gguf_get_arr_str(const struct gguf_context * ctx, int64_t key_id, size_t i) {785 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));786 GGML_ASSERT(ctx->kv[key_id].get_type() == GGUF_TYPE_STRING);787 return ctx->kv[key_id].data_string[i].c_str();788}789 790size_t gguf_get_arr_n(const struct gguf_context * ctx, int64_t key_id) {791 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));792 793 if (ctx->kv[key_id].type == GGUF_TYPE_STRING) {794 return ctx->kv[key_id].data_string.size();795 }796 797 const size_t type_size = gguf_type_size(ctx->kv[key_id].type);798 GGML_ASSERT(ctx->kv[key_id].data.size() % type_size == 0);799 return ctx->kv[key_id].data.size() / type_size;800}801 802uint8_t gguf_get_val_u8(const struct gguf_context * ctx, int64_t key_id) {803 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));804 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);805 return ctx->kv[key_id].get_val<uint8_t>();806}807 808int8_t gguf_get_val_i8(const struct gguf_context * ctx, int64_t key_id) {809 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));810 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);811 return ctx->kv[key_id].get_val<int8_t>();812}813 814uint16_t gguf_get_val_u16(const struct gguf_context * ctx, int64_t key_id) {815 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));816 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);817 return ctx->kv[key_id].get_val<uint16_t>();818}819 820int16_t gguf_get_val_i16(const struct gguf_context * ctx, int64_t key_id) {821 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));822 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);823 return ctx->kv[key_id].get_val<int16_t>();824}825 826uint32_t gguf_get_val_u32(const struct gguf_context * ctx, int64_t key_id) {827 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));828 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);829 return ctx->kv[key_id].get_val<uint32_t>();830}831 832int32_t gguf_get_val_i32(const struct gguf_context * ctx, int64_t key_id) {833 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));834 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);835 return ctx->kv[key_id].get_val<int32_t>();836}837 838float gguf_get_val_f32(const struct gguf_context * ctx, int64_t key_id) {839 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));840 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);841 return ctx->kv[key_id].get_val<float>();842}843 844uint64_t gguf_get_val_u64(const struct gguf_context * ctx, int64_t key_id) {845 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));846 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);847 return ctx->kv[key_id].get_val<uint64_t>();848}849 850int64_t gguf_get_val_i64(const struct gguf_context * ctx, int64_t key_id) {851 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));852 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);853 return ctx->kv[key_id].get_val<int64_t>();854}855 856double gguf_get_val_f64(const struct gguf_context * ctx, int64_t key_id) {857 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));858 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);859 return ctx->kv[key_id].get_val<double>();860}861 862bool gguf_get_val_bool(const struct gguf_context * ctx, int64_t key_id) {863 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));864 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);865 return ctx->kv[key_id].get_val<bool>();866}867 868const char * gguf_get_val_str(const struct gguf_context * ctx, int64_t key_id) {869 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));870 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);871 return ctx->kv[key_id].get_val<std::string>().c_str();872}873 874const void * gguf_get_val_data(const struct gguf_context * ctx, int64_t key_id) {875 GGML_ASSERT(key_id >= 0 && key_id < gguf_get_n_kv(ctx));876 GGML_ASSERT(ctx->kv[key_id].get_ne() == 1);877 GGML_ASSERT(ctx->kv[key_id].get_type() != GGUF_TYPE_STRING);878 return ctx->kv[key_id].data.data();879}880 881int64_t gguf_get_n_tensors(const struct gguf_context * ctx) {882 return ctx->info.size();883}884 885int64_t gguf_find_tensor(const struct gguf_context * ctx, const char * name) {886 // return -1 if tensor not found887 int64_t tensor_id = -1;888 889 const int64_t n_tensors = gguf_get_n_tensors(ctx);890 891 for (int64_t i = 0; i < n_tensors; ++i) {892 if (strcmp(name, gguf_get_tensor_name(ctx, i)) == 0) {893 tensor_id = i;894 break;895 }896 }897 898 return tensor_id;899}900 901size_t gguf_get_tensor_offset(const struct gguf_context * ctx, int64_t tensor_id) {902 GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));903 return ctx->info[tensor_id].offset;904}905 906const char * gguf_get_tensor_name(const struct gguf_context * ctx, int64_t tensor_id) {907 GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));908 return ctx->info[tensor_id].t.name;909}910 911enum ggml_type gguf_get_tensor_type(const struct gguf_context * ctx, int64_t tensor_id) {912 GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));913 return ctx->info[tensor_id].t.type;914}915 916size_t gguf_get_tensor_size(const struct gguf_context * ctx, int64_t tensor_id) {917 GGML_ASSERT(tensor_id >= 0 && tensor_id < gguf_get_n_tensors(ctx));918 return ggml_nbytes(&ctx->info[tensor_id].t);919}920 921int64_t gguf_remove_key(struct gguf_context * ctx, const char * key) {922 const int64_t key_id = gguf_find_key(ctx, key);923 if (key_id >= 0) {924 ctx->kv.erase(ctx->kv.begin() + key_id);925 }926 return key_id;927}928 929template<typename T>930static void gguf_check_reserved_keys(const std::string & key, const T val) {931 if (key == GGUF_KEY_GENERAL_ALIGNMENT) {932 if constexpr (std::is_same<T, uint32_t>::value) {933 GGML_ASSERT(val > 0 && (val & (val - 1)) == 0 && GGUF_KEY_GENERAL_ALIGNMENT " must be power of 2");934 } else {935 GGML_ABORT(GGUF_KEY_GENERAL_ALIGNMENT " must be type u32");936 }937 }938}939 940void gguf_set_val_u8(struct gguf_context * ctx, const char * key, uint8_t val) {941 gguf_check_reserved_keys(key, val);942 gguf_remove_key(ctx, key);943 ctx->kv.emplace_back(key, val);944}945 946void gguf_set_val_i8(struct gguf_context * ctx, const char * key, int8_t val) {947 gguf_check_reserved_keys(key, val);948 gguf_remove_key(ctx, key);949 ctx->kv.emplace_back(key, val);950}951 952void gguf_set_val_u16(struct gguf_context * ctx, const char * key, uint16_t val) {953 gguf_check_reserved_keys(key, val);954 gguf_remove_key(ctx, key);955 ctx->kv.emplace_back(key, val);956}957 958void gguf_set_val_i16(struct gguf_context * ctx, const char * key, int16_t val) {959 gguf_check_reserved_keys(key, val);960 gguf_remove_key(ctx, key);961 ctx->kv.emplace_back(key, val);962}963 964void gguf_set_val_u32(struct gguf_context * ctx, const char * key, uint32_t val) {965 gguf_check_reserved_keys(key, val);966 gguf_remove_key(ctx, key);967 ctx->kv.emplace_back(key, val);968}969 970void gguf_set_val_i32(struct gguf_context * ctx, const char * key, int32_t val) {971 gguf_check_reserved_keys(key, val);972 gguf_remove_key(ctx, key);973 ctx->kv.emplace_back(key, val);974}975 976void gguf_set_val_f32(struct gguf_context * ctx, const char * key, float val) {977 gguf_check_reserved_keys(key, val);978 gguf_remove_key(ctx, key);979 ctx->kv.emplace_back(key, val);980}981 982void gguf_set_val_u64(struct gguf_context * ctx, const char * key, uint64_t val) {983 gguf_check_reserved_keys(key, val);984 gguf_remove_key(ctx, key);985 ctx->kv.emplace_back(key, val);986}987 988void gguf_set_val_i64(struct gguf_context * ctx, const char * key, int64_t val) {989 gguf_check_reserved_keys(key, val);990 gguf_remove_key(ctx, key);991 ctx->kv.emplace_back(key, val);992}993 994void gguf_set_val_f64(struct gguf_context * ctx, const char * key, double val) {995 gguf_check_reserved_keys(key, val);996 gguf_remove_key(ctx, key);997 ctx->kv.emplace_back(key, val);998}999 1000void gguf_set_val_bool(struct gguf_context * ctx, const char * key, bool val) {1001 gguf_check_reserved_keys(key, val);1002 gguf_remove_key(ctx, key);1003 ctx->kv.emplace_back(key, val);1004}1005 1006void gguf_set_val_str(struct gguf_context * ctx, const char * key, const char * val) {1007 gguf_check_reserved_keys(key, val);1008 gguf_remove_key(ctx, key);1009 ctx->kv.emplace_back(key, std::string(val));1010}1011 1012void gguf_set_arr_data(struct gguf_context * ctx, const char * key, enum gguf_type type, const void * data, size_t n) {1013 gguf_check_reserved_keys(key, data);1014 gguf_remove_key(ctx, key);1015 1016 const size_t nbytes = n*gguf_type_size(type);1017 std::vector<int8_t> tmp(nbytes);1018 if (!tmp.empty()) {1019 memcpy(tmp.data(), data, nbytes);1020 }1021 ctx->kv.emplace_back(key, tmp);1022 ctx->kv.back().cast(type);1023}1024 1025void gguf_set_arr_str(struct gguf_context * ctx, const char * key, const char ** data, size_t n) {1026 gguf_check_reserved_keys(key, data);1027 gguf_remove_key(ctx, key);1028 1029 std::vector<std::string> tmp(n);1030 for (size_t i = 0; i < n; ++i) {1031 tmp[i] = data[i];1032 }1033 ctx->kv.emplace_back(key, tmp);1034}1035 1036// set or add KV pairs from another context1037void gguf_set_kv(struct gguf_context * ctx, const struct gguf_context * src) {1038 const int64_t n_kv = gguf_get_n_kv(src);1039 for (int64_t i = 0; i < n_kv; ++i) {1040 const struct gguf_kv & kv = src->kv[i];1041 1042 if (!kv.is_array) {1043 switch (kv.get_type()) {1044 case GGUF_TYPE_UINT8: gguf_set_val_u8 (ctx, kv.get_key().c_str(), kv.get_val<uint8_t>()); break;1045 case GGUF_TYPE_INT8: gguf_set_val_i8 (ctx, kv.get_key().c_str(), kv.get_val<int8_t>()); break;1046 case GGUF_TYPE_UINT16: gguf_set_val_u16 (ctx, kv.get_key().c_str(), kv.get_val<uint16_t>()); break;1047 case GGUF_TYPE_INT16: gguf_set_val_i16 (ctx, kv.get_key().c_str(), kv.get_val<int16_t>()); break;1048 case GGUF_TYPE_UINT32: gguf_set_val_u32 (ctx, kv.get_key().c_str(), kv.get_val<uint32_t>()); break;1049 case GGUF_TYPE_INT32: gguf_set_val_i32 (ctx, kv.get_key().c_str(), kv.get_val<int32_t>()); break;1050 case GGUF_TYPE_FLOAT32: gguf_set_val_f32 (ctx, kv.get_key().c_str(), kv.get_val<float>()); break;1051 case GGUF_TYPE_UINT64: gguf_set_val_u64 (ctx, kv.get_key().c_str(), kv.get_val<uint64_t>()); break;1052 case GGUF_TYPE_INT64: gguf_set_val_i64 (ctx, kv.get_key().c_str(), kv.get_val<int64_t>()); break;1053 case GGUF_TYPE_FLOAT64: gguf_set_val_f64 (ctx, kv.get_key().c_str(), kv.get_val<double>()); break;1054 case GGUF_TYPE_BOOL: gguf_set_val_bool(ctx, kv.get_key().c_str(), kv.get_val<bool>()); break;1055 case GGUF_TYPE_STRING: gguf_set_val_str (ctx, kv.get_key().c_str(), kv.get_val<std::string>().c_str()); break;1056 case GGUF_TYPE_ARRAY:1057 default: GGML_ABORT("invalid type");1058 }1059 continue;1060 }1061 1062 const size_t ne = kv.get_ne();1063 1064 switch (kv.get_type()) {1065 case GGUF_TYPE_UINT8:1066 case GGUF_TYPE_INT8:1067 case GGUF_TYPE_UINT16:1068 case GGUF_TYPE_INT16:1069 case GGUF_TYPE_UINT32:1070 case GGUF_TYPE_INT32:1071 case GGUF_TYPE_FLOAT32:1072 case GGUF_TYPE_UINT64:1073 case GGUF_TYPE_INT64:1074 case GGUF_TYPE_FLOAT64:1075 case GGUF_TYPE_BOOL: {1076 gguf_set_arr_data(ctx, kv.get_key().c_str(), kv.get_type(), kv.data.data(), ne);1077 } break;1078 case GGUF_TYPE_STRING: {1079 std::vector<const char *> tmp(ne);1080 for (size_t j = 0; j < ne; ++j) {1081 tmp[j] = kv.data_string[j].c_str();1082 }1083 gguf_set_arr_str(ctx, kv.get_key().c_str(), tmp.data(), ne);1084 } break;1085 case GGUF_TYPE_ARRAY:1086 default: GGML_ABORT("invalid type");1087 }1088 }1089}1090 1091void gguf_add_tensor(1092 struct gguf_context * ctx,1093 const struct ggml_tensor * tensor) {1094 GGML_ASSERT(tensor);1095 if (gguf_find_tensor(ctx, tensor->name) != -1) {1096 GGML_ABORT("duplicate tensor name: %s", tensor->name);1097 }1098 1099 struct gguf_tensor_info ti;1100 ti.t = *tensor;1101 ti.offset = ctx->info.empty() ? 0 :1102 ctx->info.back().offset + GGML_PAD(ggml_nbytes(&ctx->info.back().t), ctx->alignment);1103 ctx->info.push_back(ti);1104}1105 1106void gguf_set_tensor_type(struct gguf_context * ctx, const char * name, enum ggml_type type) {1107 const int64_t tensor_id = gguf_find_tensor(ctx, name);1108 if (tensor_id < 0) {1109 GGML_ABORT("tensor not found: %s", name);1110 }1111 struct ggml_tensor * tensor = &ctx->info[tensor_id].t;1112 const size_t type_size = ggml_type_size(type);1113 const int64_t blck_size = ggml_blck_size(type);1114 1115 tensor->type = type;1116 GGML_ASSERT(tensor->ne[0] % blck_size == 0 && "tensor row size not divisible by block size of new type");1117 1118 tensor->nb[0] = type_size;1119 tensor->nb[1] = tensor->nb[0]*(tensor->ne[0]/blck_size);1120 for (int i = 2; i < GGML_MAX_DIMS; i++) {1121 tensor->nb[i] = tensor->nb[i - 1]*tensor->ne[i - 1];1122 }1123 1124 // update offsets1125 const int64_t n_tensors = gguf_get_n_tensors(ctx);1126 for (int64_t i = tensor_id + 1; i < n_tensors; ++i) {1127 ctx->info[i].offset = ctx->info[i - 1].offset + GGML_PAD(ggml_nbytes(&ctx->info[i - 1].t), ctx->alignment);1128 }1129}1130 1131void gguf_set_tensor_data(struct gguf_context * ctx, const char * name, const void * data) {1132 const int64_t tensor_id = gguf_find_tensor(ctx, name);1133 if (tensor_id < 0) {1134 GGML_ABORT("tensor not found: %s", name);1135 }1136 1137 ctx->info[tensor_id].t.data = (void *)(uintptr_t)data; // double cast suppresses warning about casting away const1138}1139 1140struct gguf_writer {1141 std::vector<int8_t> & buf;1142 1143 gguf_writer(std::vector<int8_t> & buf) : buf(buf) {}1144 1145 template <typename T>1146 void write(const T & val) const {1147 for (size_t i = 0; i < sizeof(val); ++i) {1148 buf.push_back(reinterpret_cast<const int8_t *>(&val)[i]);1149 }1150 }1151 1152 void write(const std::vector<int8_t> & val) const {1153 buf.insert(buf.end(), val.begin(), val.end());1154 }1155 1156 void write(const bool & val) const {1157 const int8_t val8 = val ? 1 : 0;1158 write(val8);1159 }1160 1161 void write(const std::string & val) const {1162 {1163 const uint64_t n = val.length();1164 write(n);1165 }1166 for (size_t i = 0; i < val.length(); ++i) {1167 buf.push_back(reinterpret_cast<const int8_t *>(val.data())[i]);1168 }1169 }1170 1171 void write(const char * val) const {1172 write(std::string(val));1173 }1174 1175 void write(const enum ggml_type & val) const {1176 write(int32_t(val));1177 }1178 1179 void write(const enum gguf_type & val) const {1180 write(int32_t(val));1181 }1182 1183 void write(const struct gguf_kv & kv) const {1184 const uint64_t ne = kv.get_ne();1185 1186 write(kv.get_key());1187 1188 if (kv.is_array) {1189 write(GGUF_TYPE_ARRAY);1190 write(kv.get_type());1191 write(ne);1192 } else {1193 write(kv.get_type());1194 }1195 1196 switch (kv.get_type()) {1197 case GGUF_TYPE_UINT8:1198 case GGUF_TYPE_INT8:1199 case GGUF_TYPE_UINT16:1200 case GGUF_TYPE_INT16: