KBaba7/llama.cpp
0
1#include "ggml.h"2#include "gguf.h"3 4#include <cstdlib> /* abort() */5#include <cstddef>6#include <cstdio>7#include <string>8#include <stdexcept>9#include <algorithm>10#include <cstring>11 12#include <sstream>13#include <fstream>14 15#ifdef __cplusplus16extern "C" {17#endif18 19#include "xxhash/xxhash.h"20#include "sha1/sha1.h"21#include "sha256/sha256.h"22 23#ifdef __cplusplus24}25#endif26 27 28// uuid.uuid5(uuid.NAMESPACE_URL, 'en.wikipedia.org/wiki/Llama.cpp')29#define UUID_NAMESPACE_LLAMA_CPP "ef001206-dadc-5f6d-a15f-3359e577d4e5"30#define UUID_NAMESPACE_LLAMA_CPP_HEX 0xef, 0x00, 0x12, 0x06, 0xda, 0xdc, 0x5f, 0x6d, 0xa1, 0x5f, 0x33, 0x59, 0xe5, 0x77, 0xd4, 0xe531 32 33#define HASH_TYPE_SHA256_STR "sha256"34#define HASH_TYPE_SHA1_STR "sha1"35#define HASH_TYPE_XXH64_STR "xxh64"36#define HASH_TYPE_UUID_STR "uuid"37 38 39typedef enum {40 HASH_EXIT_SUCCESS = 0, // All hash has been generated or validated41 HASH_EXIT_FAILURE = 1, // Generic Failure42 HASH_EXIT_MISMATCH = 2, // Hash mismatched during validation43 HASH_EXIT_MANIFEST_MISSING_ENTRY = 3, // Hash attempted validation but missing entry in manifest44 HASH_EXIT_MANIFEST_UNKNOWN_HASH = 4, // Manifest is present, but we do not know any hash format within it45 HASH_EXIT_MANIFEST_FILE_ERROR = 5 // Manifest is either missing or not a known format46} hash_exit_code_t;47 48 49typedef enum {50 HASH_MANIFEST_NOT_FOUND,51 HASH_MANIFEST_MISMATCH,52 HASH_MANIFEST_OK,53} hash_manifest_result_t;54 55 56struct hash_params {57 std::string input;58 bool xxh64 = false;59 bool sha1 = false;60 bool sha256 = false;61 bool uuid = false;62 63 bool no_layer = false;64 65 bool manifest_is_usable = false;66 std::string manifest_file;67};68 69struct manifest_check_params {70 bool xxh64 = false;71 bool sha1 = false;72 bool sha256 = false;73 bool uuid = false;74};75 76static char const * hash_manifest_result_to_str(hash_manifest_result_t value) {77 switch (value) {78 case HASH_MANIFEST_NOT_FOUND: return "Not Found";79 case HASH_MANIFEST_MISMATCH: return "Mismatch";80 case HASH_MANIFEST_OK: return "Ok";81 }82 return "?";83}84 85static char const * hash_exit_code_to_str(hash_exit_code_t value) {86 switch (value) {87 case HASH_EXIT_SUCCESS: return "Success";88 case HASH_EXIT_FAILURE: return "Failure";89 case HASH_EXIT_MISMATCH: return "Mismatch";90 case HASH_EXIT_MANIFEST_MISSING_ENTRY: return "Manifest Missing Entry";91 case HASH_EXIT_MANIFEST_UNKNOWN_HASH: return "Manifest Unknown Hash";92 case HASH_EXIT_MANIFEST_FILE_ERROR: return "Manifest File Error";93 }94 return "?";95}96 97static void hash_print_usage(const char * executable) {98 const hash_params default_params;99 printf("\n");100 printf("usage: %s [options] GGUF_IN\n", executable);101 printf("\n");102 printf("Hash a GGUF file");103 printf("\n");104 printf("options:\n");105 printf(" -h, --help show this help message and exit\n");106 printf(" --xxh64 use xxh64 hash\n");107 printf(" --sha1 use sha1 hash\n");108 printf(" --sha256 use sha256 hash\n");109 printf(" --all use all hash\n");110 printf(" --no-layer exclude per layer hash\n");111 printf(" --uuid generate UUIDv5 ID\n");112 printf(" -c, --check <manifest> verify against a manifest\n");113 printf("\n");114}115 116static void hash_params_parse_ex(int argc, const char ** argv, hash_params & params) {117 std::string arg;118 bool invalid_param = false;119 const std::string arg_prefix = "--";120 121 int arg_idx = 1;122 for (; arg_idx < argc && strncmp(argv[arg_idx], "--", 2) == 0; arg_idx++) {123 arg = argv[arg_idx];124 if (arg.compare(0, arg_prefix.size(), arg_prefix) == 0) {125 std::replace(arg.begin(), arg.end(), '_', '-');126 }127 128 bool arg_found = false;129 if (arg == "-h" || arg == "--help") {130 hash_print_usage(argv[0]);131 exit(0);132 }133 134 if (arg == "--xxh64") {135 arg_found = true;136 params.xxh64 = true;137 }138 139 if (arg == "--sha1") {140 arg_found = true;141 params.sha1 = true;142 }143 144 if (arg == "--uuid") {145 arg_found = true;146 params.uuid = true;147 }148 149 if (arg == "--sha256") {150 arg_found = true;151 params.sha256 = true;152 }153 154 if (arg == "--all") {155 arg_found = true;156 params.sha256 = true;157 params.sha1 = true;158 params.xxh64 = true;159 }160 161 if (arg == "--no-layer") {162 arg_found = true;163 params.no_layer = true;164 }165 166 if (arg == "-c" || arg == "--check") {167 if (++arg_idx >= argc) {168 invalid_param = true;169 break;170 }171 arg_found = true;172 params.manifest_file = argv[arg_idx];173 }174 175 if (!arg_found) {176 throw std::invalid_argument("error: unknown argument: " + arg);177 }178 }179 180 if (invalid_param) {181 throw std::invalid_argument("error: invalid parameter for argument:" + arg);182 }183 184 if (argc - arg_idx < 1) {185 throw std::invalid_argument("error: bad arguments");186 }187 188 params.input = argv[arg_idx++];189}190 191static bool hash_params_parse(int argc, const char ** argv, hash_params & params) {192 bool result = true;193 try {194 hash_params_parse_ex(argc, argv, params);195 }196 catch (const std::invalid_argument & ex) {197 fprintf(stderr, "%s\n", ex.what());198 hash_print_usage(argv[0]);199 exit(EXIT_FAILURE);200 }201 return result;202}203 204static bool manifest_type(const std::string & manifest_file, manifest_check_params & manifest_check) {205 if (manifest_file.empty()) {206 return false;207 }208 209 std::ifstream file(manifest_file);210 if (!file.is_open()) {211 return false;212 }213 214 std::string manifest_entry_line;215 while (getline(file, manifest_entry_line)) {216 // hash_type_str hash_str tensor_name217 // e.g. 'xxh64 f66e9cd66a4396a0 test.gguf:tensor_0'218 std::istringstream line_stream(manifest_entry_line);219 std::string file_hash_type;220 if (line_stream >> file_hash_type) {221 if (file_hash_type == HASH_TYPE_SHA256_STR) {222 manifest_check.sha256 = true;223 } else if (file_hash_type == HASH_TYPE_SHA1_STR) {224 manifest_check.sha1 = true;225 } else if (file_hash_type == HASH_TYPE_XXH64_STR) {226 manifest_check.xxh64 = true;227 } else if (file_hash_type == HASH_TYPE_UUID_STR) {228 manifest_check.uuid = true;229 }230 }231 }232 233 return true;234}235 236static hash_manifest_result_t manifest_verify(const std::string& manifest_file, const std::string& hash_type_str, const std::string& hash_str, const std::string& tensor_name) {237 if (manifest_file.empty()) {238 return HASH_MANIFEST_NOT_FOUND;239 }240 241 std::ifstream file(manifest_file);242 if (!file.is_open()) {243 return HASH_MANIFEST_NOT_FOUND;244 }245 246 std::string manifest_entry_line;247 while (getline(file, manifest_entry_line)) {248 std::istringstream line_stream(manifest_entry_line);249 std::string file_hash_type;250 std::string file_hash;251 std::string file_tensor_name;252 if (line_stream >> file_hash_type >> file_hash >> file_tensor_name) {253 // Line parsed. Check hash validity254 255 if (file_hash_type != hash_type_str) {256 continue;257 }258 259 if (file_tensor_name != tensor_name) {260 continue;261 }262 263 return (file_hash == hash_str) ? HASH_MANIFEST_OK : HASH_MANIFEST_MISMATCH;264 }265 }266 267 return HASH_MANIFEST_NOT_FOUND;268}269 270static void generate_uuidv5(const unsigned char sha1_digest[20], unsigned char uuid[16]) {271 // Ref: https://www.rfc-editor.org/rfc/rfc9562.html#section-5.5272 // Assumes that digest was processed correctly with the expected namespace273 for (int i = 0; i < 16; i++) {274 uuid[i] = sha1_digest[i];275 }276 277 // Set bits corresponding to UUID ver 5278 uuid[ 6] &= ~(0xF << 4);279 uuid[ 6] |= (5 << 4);280 281 // Set bits corresponding to UUID variant 0b10XX282 uuid[ 8] &= ~(0xc << 4);283 uuid[ 8] |= (0x8 << 4);284}285 286static hash_exit_code_t gguf_hash(const hash_params & hash_params) {287 const std::string & fname = hash_params.input;288 struct ggml_context * ctx_data = NULL;289 290 struct gguf_init_params params = {291 /*.no_alloc = */ false,292 /*.ctx = */ &ctx_data,293 };294 295 // xxh64 init296 XXH64_state_t* xxh64_model_hash_state = NULL;297 if (hash_params.xxh64) {298 xxh64_model_hash_state = XXH64_createState();299 if (xxh64_model_hash_state==NULL) {300 abort();301 }302 303 XXH64_hash_t const seed = 0;304 if (XXH64_reset(xxh64_model_hash_state, seed) == XXH_ERROR) {305 abort();306 }307 }308 309 // sha1 init310 SHA1_CTX sha1_model_hash_ctx;311 if (hash_params.sha1) {312 SHA1Init(&sha1_model_hash_ctx);313 }314 315 // sha256 init316 sha256_t sha256_model_hash_ctx;317 if (hash_params.sha256) {318 sha256_init(&sha256_model_hash_ctx);319 }320 321 // sha1 for uuid init322 SHA1_CTX sha1_for_uuid_ctx;323 if (hash_params.uuid) {324 unsigned char const uuidv5_namespace[] = {UUID_NAMESPACE_LLAMA_CPP_HEX};325 SHA1Init(&sha1_for_uuid_ctx);326 SHA1Update( &sha1_for_uuid_ctx, (unsigned char const *)uuidv5_namespace, sizeof(uuidv5_namespace));327 }328 329 struct gguf_context * ctx = gguf_init_from_file(fname.c_str(), params);330 const int n_tensors = gguf_get_n_tensors(ctx);331 bool tensor_layer_in_manifest = false;332 bool model_in_manifest = false;333 bool tensor_layer_has_mismatch = false;334 bool model_has_mismatch = false;335 for (int i = 0; i < n_tensors; ++i) {336 const char * name = gguf_get_tensor_name(ctx, i);337 struct ggml_tensor * cur = ggml_get_tensor(ctx_data, name);338 auto n_bytes = ggml_nbytes(cur);339 auto *raw_data = cur->data;340 const std::string tensor_layer_name = fname + ":" + name;341 342 if (hash_params.xxh64) {343 344 if (!hash_params.no_layer) {345 // Per Layer Hash346 XXH64_hash_t hash = XXH64(raw_data, n_bytes, 0);347 348 char hex_result[17];349 for (int offset = 0; offset < 8; offset++) {350 unsigned int shift_bits_by = (8 * (8 - offset - 1));351 snprintf( ( hex_result + (2*offset)), sizeof(hex_result) - (2*offset), "%02x", (unsigned char) (hash >> shift_bits_by)&0xff);352 }353 354 if (hash_params.manifest_is_usable) {355 hash_manifest_result_t verify_result = manifest_verify(hash_params.manifest_file, HASH_TYPE_XXH64_STR, hex_result, tensor_layer_name);356 357 switch (verify_result) {358 case HASH_MANIFEST_NOT_FOUND:359 break;360 case HASH_MANIFEST_MISMATCH:361 tensor_layer_in_manifest = true;362 tensor_layer_has_mismatch = true;363 break;364 case HASH_MANIFEST_OK:365 tensor_layer_in_manifest = true;366 break;367 }368 369 printf("%-8s %-s %s - %s\n", HASH_TYPE_XXH64_STR, hex_result, tensor_layer_name.c_str(), hash_manifest_result_to_str(verify_result));370 } else {371 printf("%-8s %-s %s\n", HASH_TYPE_XXH64_STR, hex_result, tensor_layer_name.c_str());372 }373 }374 375 // Overall Model Hash376 if (XXH64_update(xxh64_model_hash_state, raw_data, n_bytes) == XXH_ERROR) abort();377 }378 379 if (hash_params.sha1) {380 381 if (!hash_params.no_layer) {382 // Per Layer Hash383 char result[21]; // sha1 outputs 20 bytes384 SHA1( result, (const char *)raw_data, n_bytes);385 386 char hex_result[41] = {0};387 for (int offset = 0; offset < 20; offset++) {388 snprintf( ( hex_result + (2*offset)), sizeof(hex_result) - (2*offset), "%02x", result[offset]&0xff);389 }390 391 if (hash_params.manifest_is_usable) {392 hash_manifest_result_t verify_result = manifest_verify(hash_params.manifest_file, HASH_TYPE_SHA1_STR, hex_result, tensor_layer_name);393 394 switch (verify_result) {395 case HASH_MANIFEST_NOT_FOUND:396 break;397 case HASH_MANIFEST_MISMATCH:398 tensor_layer_in_manifest = true;399 tensor_layer_has_mismatch = true;400 break;401 case HASH_MANIFEST_OK:402 tensor_layer_in_manifest = true;403 break;404 }405 406 printf("%-8s %-s %s - %s\n", HASH_TYPE_SHA1_STR, hex_result, tensor_layer_name.c_str(), hash_manifest_result_to_str(verify_result));407 } else {408 printf("%-8s %-s %s\n", HASH_TYPE_SHA1_STR, hex_result, tensor_layer_name.c_str());409 }410 }411 412 // Overall Model Hash413 SHA1Update( &sha1_model_hash_ctx, (unsigned char const *)raw_data, n_bytes);414 }415 416 if (hash_params.sha256) {417 418 if (!hash_params.no_layer) {419 // Per Layer Hash420 unsigned char result[SHA256_DIGEST_SIZE]; // sha256 outputs 32 bytes421 sha256_hash((unsigned char*) result, (const unsigned char *)raw_data, n_bytes);422 423 char hex_result[SHA256_DIGEST_SIZE * 2 + 1] = {0};424 for (int offset = 0; offset < SHA256_DIGEST_SIZE; offset++) {425 snprintf( ( hex_result + (2*offset)), sizeof(hex_result) - (2*offset), "%02x", result[offset]&0xff);426 }427 428 if (hash_params.manifest_is_usable) {429 hash_manifest_result_t verify_result = manifest_verify(hash_params.manifest_file, HASH_TYPE_SHA256_STR, hex_result, tensor_layer_name);430 431 switch (verify_result) {432 case HASH_MANIFEST_NOT_FOUND:433 break;434 case HASH_MANIFEST_MISMATCH:435 tensor_layer_in_manifest = true;436 tensor_layer_has_mismatch = true;437 break;438 case HASH_MANIFEST_OK:439 tensor_layer_in_manifest = true;440 break;441 }442 443 printf("%-8s %-s %s - %s\n", HASH_TYPE_SHA256_STR, hex_result, tensor_layer_name.c_str(), hash_manifest_result_to_str(verify_result));444 } else {445 printf("%-8s %-s %s\n", HASH_TYPE_SHA256_STR, hex_result, tensor_layer_name.c_str());446 }447 }448 449 // Overall Model Hash450 sha256_update( &sha256_model_hash_ctx, (unsigned char const *)raw_data, n_bytes);451 }452 453 if (hash_params.uuid) {454 SHA1Update( &sha1_for_uuid_ctx, (unsigned char const *)raw_data, n_bytes);455 }456 }457 458 if (hash_params.xxh64) {459 XXH64_hash_t const hash = XXH64_digest(xxh64_model_hash_state);460 461 char hex_result[17];462 for (int offset = 0; offset < 8; offset++) {463 unsigned int shift_bits_by = (8 * (8 - offset - 1));464 snprintf( ( hex_result + (2*offset)), sizeof(hex_result) - (2*offset), "%02x", (unsigned char) (hash >> shift_bits_by)&0xff);465 }466 467 if (hash_params.manifest_is_usable) {468 hash_manifest_result_t verify_result = manifest_verify(hash_params.manifest_file, HASH_TYPE_XXH64_STR, hex_result, fname);469 470 switch (verify_result) {471 case HASH_MANIFEST_NOT_FOUND:472 break;473 case HASH_MANIFEST_MISMATCH:474 model_in_manifest = true;475 model_has_mismatch = true;476 break;477 case HASH_MANIFEST_OK:478 model_in_manifest = true;479 break;480 }481 482 printf("%-8s %-s %s - %s\n", HASH_TYPE_XXH64_STR, hex_result, fname.c_str(), hash_manifest_result_to_str(verify_result));483 } else {484 printf("%-8s %-s %s\n", HASH_TYPE_XXH64_STR, hex_result, fname.c_str());485 }486 }487 488 if (hash_params.sha1) {489 unsigned char result[21];490 SHA1Final(result, &sha1_model_hash_ctx);491 492 char hex_result[41];493 for (int offset = 0; offset < 20; offset++) {494 snprintf( ( hex_result + (2*offset)), sizeof(hex_result) - (2*offset), "%02x", result[offset]&0xff);495 }496 497 if (hash_params.manifest_is_usable) {498 hash_manifest_result_t verify_result = manifest_verify(hash_params.manifest_file, HASH_TYPE_SHA1_STR, hex_result, fname);499 500 switch (verify_result) {501 case HASH_MANIFEST_NOT_FOUND:502 break;503 case HASH_MANIFEST_MISMATCH:504 model_in_manifest = true;505 model_has_mismatch = true;506 break;507 case HASH_MANIFEST_OK:508 model_in_manifest = true;509 break;510 }511 512 printf("%-8s %-s %s - %s\n", HASH_TYPE_SHA1_STR, hex_result, fname.c_str(), hash_manifest_result_to_str(verify_result));513 } else {514 printf("%-8s %-s %s\n", HASH_TYPE_SHA1_STR, hex_result, fname.c_str());515 }516 }517 518 if (hash_params.sha256) {519 unsigned char result[SHA256_DIGEST_SIZE]; // sha256 outputs 32 bytes520 sha256_final( &sha256_model_hash_ctx, result);521 522 char hex_result[SHA256_DIGEST_SIZE * 2 + 1] = {0};523 for (int offset = 0; offset < SHA256_DIGEST_SIZE; offset++) {524 snprintf( ( hex_result + (2*offset)), sizeof(hex_result) - (2*offset), "%02x", result[offset]&0xff);525 }526 527 if (hash_params.manifest_is_usable) {528 hash_manifest_result_t verify_result = manifest_verify(hash_params.manifest_file, HASH_TYPE_SHA256_STR, hex_result, fname);529 530 switch (verify_result) {531 case HASH_MANIFEST_NOT_FOUND:532 break;533 case HASH_MANIFEST_MISMATCH:534 model_in_manifest = true;535 model_has_mismatch = true;536 break;537 case HASH_MANIFEST_OK:538 model_in_manifest = true;539 break;540 }541 542 printf("%-8s %-s %s - %s\n", HASH_TYPE_SHA256_STR, hex_result, fname.c_str(), hash_manifest_result_to_str(verify_result));543 } else {544 printf("%-8s %-s %s\n", HASH_TYPE_SHA256_STR, hex_result, fname.c_str());545 }546 }547 548 if (hash_params.uuid) {549 unsigned char result[21];550 SHA1Final(result, &sha1_for_uuid_ctx);551 552 unsigned char uuid[16];553 generate_uuidv5(result, uuid);554 555 char string_buffer[37] = {0};556 snprintf(string_buffer, sizeof(string_buffer), "%02x%02x%02x%02x-%02x%02x-%02x%02x-%02x%02x-%02x%02x%02x%02x%02x%02x",557 uuid[0], uuid[1], uuid[2], uuid[3],558 uuid[4], uuid[5], uuid[6], uuid[7],559 uuid[8], uuid[9], uuid[10], uuid[11],560 uuid[12], uuid[13], uuid[14], uuid[15]);561 562 if (hash_params.manifest_is_usable) {563 hash_manifest_result_t verify_result = manifest_verify(hash_params.manifest_file, HASH_TYPE_SHA256_STR, string_buffer, fname);564 565 switch (verify_result) {566 case HASH_MANIFEST_NOT_FOUND:567 break;568 case HASH_MANIFEST_MISMATCH:569 model_in_manifest = true;570 model_has_mismatch = true;571 break;572 case HASH_MANIFEST_OK:573 model_in_manifest = true;574 break;575 }576 577 printf("%-8s %-s %s - %s\n", HASH_TYPE_UUID_STR, string_buffer, fname.c_str(), hash_manifest_result_to_str(verify_result));578 } else {579 printf("%-8s %-s %s\n", HASH_TYPE_UUID_STR, string_buffer, fname.c_str());580 }581 }582 583 584 ggml_free(ctx_data);585 gguf_free(ctx);586 587 588 if (hash_params.manifest_is_usable) {589 // In hash verification mode590 591 if (!model_in_manifest) {592 // model missing in manifest?593 594 // Check tensor layer...595 if (!tensor_layer_in_manifest) {596 // Still missing? Maybe we are reading the wrong manifest.597 return HASH_EXIT_MANIFEST_MISSING_ENTRY;598 }599 600 if (tensor_layer_has_mismatch) {601 // Per tensor check found error602 return HASH_EXIT_FAILURE;603 }604 605 // All per tensor layer checks passed? Sounds good enough.606 return HASH_EXIT_SUCCESS;607 }608 609 // Overall model check passed, but let's check per layer just in case610 // If missing, we don't care too much as the overall model checked611 if (tensor_layer_in_manifest && tensor_layer_has_mismatch) {612 return HASH_EXIT_FAILURE;613 }614 615 if (model_has_mismatch) {616 // model has failed hash somewhere in the model617 return HASH_EXIT_FAILURE;618 }619 620 // All checks appears to be fine621 return HASH_EXIT_SUCCESS;622 }623 624 // In hash generation mode625 return HASH_EXIT_SUCCESS;626}627 628int main(int argc, const char ** argv) {629 hash_params params;630 manifest_check_params manifest_check;631 hash_params_parse(argc, argv, params);632 633 if (!params.manifest_file.empty()) {634 if (!manifest_type(params.manifest_file, manifest_check)) {635 printf("ERROR cannot open manifest %s", params.manifest_file.c_str());636 return HASH_EXIT_MANIFEST_FILE_ERROR;637 }638 639 if (!manifest_check.sha256 && !manifest_check.sha1 && !manifest_check.xxh64 && !manifest_check.uuid) {640 printf("ERROR manifest does not have any known hash format in %s", params.manifest_file.c_str());641 return HASH_EXIT_MANIFEST_UNKNOWN_HASH;642 }643 644 printf("manifest %s", params.manifest_file.c_str());645 646 if (manifest_check.sha256) {647 printf(" sha256");648 }649 650 if (manifest_check.sha1) {651 printf(" sha1");652 }653 654 if (manifest_check.xxh64) {655 printf(" xxh64");656 }657 658 if (manifest_check.uuid) {659 printf(" uuid");660 }661 662 printf("\n");663 664 // Autoselect the highest security hash if manifest is provided but665 // the user has not specifically defined the hash they care about666 if (!params.xxh64 && !params.sha1 && !params.uuid && !params.sha256) {667 // User has not selected a specific value, pick most secure hash668 if (manifest_check.sha256) {669 params.sha256 = true;670 } else if (manifest_check.sha1) {671 params.sha1 = true;672 } else if (manifest_check.xxh64) {673 params.xxh64 = true;674 } else if (manifest_check.uuid) {675 params.uuid = true;676 }677 }678 679 params.manifest_is_usable = true;680 }681 682 // By default if no swich argument provided, assume xxh64683 if (!params.xxh64 && !params.sha1 && !params.uuid && !params.sha256) {684 params.xxh64 = true;685 }686 687 hash_exit_code_t exit_code = gguf_hash(params);688 689 if (params.manifest_is_usable) {690 printf("\nVerification results for %s - %s\n", params.manifest_file.c_str(), hash_exit_code_to_str(exit_code));691 }692 693 return exit_code;694}695