KBaba7/llama.cpp
0
1#include "arg.h"2#include "base64.hpp"3#include "log.h"4#include "common.h"5#include "sampling.h"6#include "clip.h"7#include "llava.h"8#include "llama.h"9#include "ggml.h"10 11#ifdef GGML_USE_CUDA12#include "ggml-cuda.h"13#endif14#ifdef NDEBUG15#include "ggml-alloc.h"16#include "ggml-backend.h"17#endif18 19#include <cstdio>20#include <cstdlib>21#include <cstring>22#include <vector>23#include <algorithm>24#include <iostream>25#include <fstream>26 27 28static bool qwen2vl_eval_image_embed(llama_context * ctx_llama, const struct llava_image_embed * image_embed,29 int n_batch, int * n_past, int * st_pos_id, struct clip_image_size * image_size) {30 int n_embd = llama_model_n_embd(llama_get_model(ctx_llama));31 const int patch_size = 14 * 2;32 const int ph = image_size->height / patch_size + (image_size->height % patch_size > 0);33 const int pw = image_size->width / patch_size + (image_size->width % patch_size > 0);34 auto img_tokens = image_embed->n_image_pos;35 // llama_pos mrope_pos[img_tokens * 4];36 std::vector<llama_pos> mrope_pos;37 mrope_pos.resize(img_tokens * 4);38 39 for (int y = 0; y < ph; y++)40 {41 for (int x = 0; x < pw; x++)42 {43 int i = y * pw + x;44 mrope_pos[i] = *st_pos_id;45 mrope_pos[i + img_tokens] = *st_pos_id + y;46 mrope_pos[i + img_tokens * 2] = *st_pos_id + x;47 mrope_pos[i + img_tokens * 3] = 0;48 }49 }50 *st_pos_id += std::max(pw, ph);51 52 int processed = 0;53 std::vector<llama_pos> batch_mrope_pos;54 batch_mrope_pos.resize(img_tokens * 4);55 56 for (int i = 0; i < img_tokens; i += n_batch) {57 int n_eval = img_tokens - i;58 if (n_eval > n_batch) {59 n_eval = n_batch;60 }61 62 // llama_pos batch_mrope_pos[n_eval * 4];63 std::fill(batch_mrope_pos.begin(), batch_mrope_pos.end(), 0);64 memcpy(batch_mrope_pos.data(), &mrope_pos[processed], n_eval * sizeof(llama_pos));65 memcpy(&batch_mrope_pos[n_eval * 1], &mrope_pos[img_tokens * 1 + processed], n_eval * sizeof(llama_pos));66 memcpy(&batch_mrope_pos[n_eval * 2], &mrope_pos[img_tokens * 2 + processed], n_eval * sizeof(llama_pos));67 memcpy(&batch_mrope_pos[n_eval * 3], &mrope_pos[img_tokens * 3 + processed], n_eval * sizeof(llama_pos));68 69 llama_batch batch = {70 int32_t(n_eval), // n_tokens71 nullptr, // token72 (image_embed->embed+i*n_embd), // embed73 batch_mrope_pos.data(), // pos74 nullptr, // n_seq_id75 nullptr, // seq_id76 nullptr, // logits77 };78 79 if (llama_decode(ctx_llama, batch)) {80 LOG_ERR("%s : failed to eval\n", __func__);81 return false;82 }83 *n_past += n_eval;84 processed += n_eval;85 }86 return true;87}88 89 90static bool eval_tokens(struct llama_context * ctx_llama, std::vector<llama_token> tokens, int n_batch, int * n_past, int * st_pos_id) {91 int N = (int) tokens.size();92 std::vector<llama_pos> pos;93 for (int i = 0; i < N; i += n_batch) {94 int n_eval = (int) tokens.size() - i;95 if (n_eval > n_batch) {96 n_eval = n_batch;97 }98 auto batch = llama_batch_get_one(&tokens[i], n_eval);99 // TODO: add mrope pos ids somewhere else100 pos.resize(batch.n_tokens * 4);101 std::fill(pos.begin(), pos.end(), 0);102 for (int j = 0; j < batch.n_tokens * 3; j ++) {103 pos[j] = *st_pos_id + (j % batch.n_tokens);104 }105 batch.pos = pos.data();106 107 if (llama_decode(ctx_llama, batch)) {108 LOG_ERR("%s : failed to eval. token %d/%d (batch size %d, n_past %d)\n", __func__, i, N, n_batch, *n_past);109 return false;110 }111 *n_past += n_eval;112 *st_pos_id += n_eval;113 }114 return true;115}116 117static bool eval_id(struct llama_context * ctx_llama, int id, int * n_past, int * st_pos_id) {118 std::vector<llama_token> tokens;119 tokens.push_back(id);120 return eval_tokens(ctx_llama, tokens, 1, n_past, st_pos_id);121}122 123static bool eval_string(struct llama_context * ctx_llama, const char* str, int n_batch, int * n_past, int * st_pos_id, bool add_bos){124 std::string str2 = str;125 std::vector<llama_token> embd_inp = common_tokenize(ctx_llama, str2, add_bos, true);126 eval_tokens(ctx_llama, embd_inp, n_batch, n_past, st_pos_id);127 return true;128}129 130static const char * sample(struct common_sampler * smpl,131 struct llama_context * ctx_llama,132 int * n_past, int * st_pos_id) {133 const llama_token id = common_sampler_sample(smpl, ctx_llama, -1);134 common_sampler_accept(smpl, id, true);135 136 const llama_model * model = llama_get_model(ctx_llama);137 const llama_vocab * vocab = llama_model_get_vocab(model);138 139 static std::string ret;140 if (llama_vocab_is_eog(vocab, id)) {141 ret = "</s>";142 } else {143 ret = common_token_to_piece(ctx_llama, id);144 }145 eval_id(ctx_llama, id, n_past, st_pos_id);146 return ret.c_str();147}148 149static const char* IMG_BASE64_TAG_BEGIN = "<img src=\"data:image/jpeg;base64,";150static const char* IMG_BASE64_TAG_END = "\">";151 152static void find_image_tag_in_prompt(const std::string& prompt, size_t& begin_out, size_t& end_out) {153 begin_out = prompt.find(IMG_BASE64_TAG_BEGIN);154 end_out = prompt.find(IMG_BASE64_TAG_END, (begin_out == std::string::npos) ? 0UL : begin_out);155}156 157static bool prompt_contains_image(const std::string& prompt) {158 size_t begin, end;159 find_image_tag_in_prompt(prompt, begin, end);160 return (begin != std::string::npos);161}162 163// replaces the base64 image tag in the prompt with `replacement`164static llava_image_embed * llava_image_embed_make_with_prompt_base64(struct clip_ctx * ctx_clip, int n_threads, const std::string& prompt) {165 size_t img_base64_str_start, img_base64_str_end;166 find_image_tag_in_prompt(prompt, img_base64_str_start, img_base64_str_end);167 if (img_base64_str_start == std::string::npos || img_base64_str_end == std::string::npos) {168 LOG_ERR("%s: invalid base64 image tag. must be %s<base64 byte string>%s\n", __func__, IMG_BASE64_TAG_BEGIN, IMG_BASE64_TAG_END);169 return NULL;170 }171 172 auto base64_bytes_start = img_base64_str_start + strlen(IMG_BASE64_TAG_BEGIN);173 auto base64_bytes_count = img_base64_str_end - base64_bytes_start;174 auto base64_str = prompt.substr(base64_bytes_start, base64_bytes_count );175 176 auto required_bytes = base64::required_encode_size(base64_str.size());177 auto img_bytes = std::vector<unsigned char>(required_bytes);178 base64::decode(base64_str.begin(), base64_str.end(), img_bytes.begin());179 180 auto embed = llava_image_embed_make_with_bytes(ctx_clip, n_threads, img_bytes.data(), img_bytes.size());181 if (!embed) {182 LOG_ERR("%s: could not load image from base64 string.\n", __func__);183 return NULL;184 }185 186 return embed;187}188 189static std::string remove_image_from_prompt(const std::string& prompt, const char * replacement = "") {190 size_t begin, end;191 find_image_tag_in_prompt(prompt, begin, end);192 if (begin == std::string::npos || end == std::string::npos) {193 return prompt;194 }195 auto pre = prompt.substr(0, begin);196 auto post = prompt.substr(end + strlen(IMG_BASE64_TAG_END));197 return pre + replacement + post;198}199 200struct llava_context {201 struct clip_ctx * ctx_clip = NULL;202 struct llama_context * ctx_llama = NULL;203 struct llama_model * model = NULL;204};205 206static void print_usage(int, char ** argv) {207 LOG("\n example usage:\n");208 LOG("\n %s -m <llava-v1.5-7b/ggml-model-q5_k.gguf> --mmproj <llava-v1.5-7b/mmproj-model-f16.gguf> --image <path/to/an/image.jpg> --image <path/to/another/image.jpg> [--temp 0.1] [-p \"describe the image in detail.\"]\n", argv[0]);209 LOG("\n note: a lower temperature value like 0.1 is recommended for better quality.\n");210}211 212static struct llava_image_embed * load_image(llava_context * ctx_llava, common_params * params, const std::string & fname) {213 214 // load and preprocess the image215 llava_image_embed * embed = NULL;216 auto prompt = params->prompt;217 if (prompt_contains_image(prompt)) {218 if (!params->image.empty()) {219 LOG_INF("using base64 encoded image instead of command line image path\n");220 }221 embed = llava_image_embed_make_with_prompt_base64(ctx_llava->ctx_clip, params->cpuparams.n_threads, prompt);222 if (!embed) {223 LOG_ERR("%s: can't load image from prompt\n", __func__);224 return NULL;225 }226 params->prompt = remove_image_from_prompt(prompt);227 } else {228 embed = llava_image_embed_make_with_filename(ctx_llava->ctx_clip, params->cpuparams.n_threads, fname.c_str());229 if (!embed) {230 fprintf(stderr, "%s: is %s really an image file?\n", __func__, fname.c_str());231 return NULL;232 }233 }234 235 return embed;236}237 238static void process_prompt(struct llava_context * ctx_llava, struct llava_image_embed * image_embed, common_params * params, const std::string & prompt) {239 int n_past = 0;240 int cur_pos_id = 0;241 242 const int max_tgt_len = params->n_predict < 0 ? 256 : params->n_predict;243 244 std::string system_prompt, user_prompt;245 size_t image_pos = prompt.find("<|vision_start|>");246 if (image_pos != std::string::npos) {247 // new templating mode: Provide the full prompt including system message and use <image> as a placeholder for the image248 system_prompt = prompt.substr(0, image_pos);249 user_prompt = prompt.substr(image_pos + std::string("<|vision_pad|>").length());250 LOG_INF("system_prompt: %s\n", system_prompt.c_str());251 if (params->verbose_prompt) {252 auto tmp = common_tokenize(ctx_llava->ctx_llama, system_prompt, true, true);253 for (int i = 0; i < (int) tmp.size(); i++) {254 LOG_INF("%6d -> '%s'\n", tmp[i], common_token_to_piece(ctx_llava->ctx_llama, tmp[i]).c_str());255 }256 }257 LOG_INF("user_prompt: %s\n", user_prompt.c_str());258 if (params->verbose_prompt) {259 auto tmp = common_tokenize(ctx_llava->ctx_llama, user_prompt, true, true);260 for (int i = 0; i < (int) tmp.size(); i++) {261 LOG_INF("%6d -> '%s'\n", tmp[i], common_token_to_piece(ctx_llava->ctx_llama, tmp[i]).c_str());262 }263 }264 } else {265 // llava-1.5 native mode266 system_prompt = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n<|vision_start|>";267 user_prompt = "<|vision_end|>" + prompt + "<|im_end|>\n<|im_start|>assistant\n";268 if (params->verbose_prompt) {269 auto tmp = common_tokenize(ctx_llava->ctx_llama, user_prompt, true, true);270 for (int i = 0; i < (int) tmp.size(); i++) {271 LOG_INF("%6d -> '%s'\n", tmp[i], common_token_to_piece(ctx_llava->ctx_llama, tmp[i]).c_str());272 }273 }274 }275 276 eval_string(ctx_llava->ctx_llama, system_prompt.c_str(), params->n_batch, &n_past, &cur_pos_id, true);277 if (image_embed != nullptr) {278 auto image_size = clip_get_load_image_size(ctx_llava->ctx_clip);279 qwen2vl_eval_image_embed(ctx_llava->ctx_llama, image_embed, params->n_batch, &n_past, &cur_pos_id, image_size);280 }281 eval_string(ctx_llava->ctx_llama, user_prompt.c_str(), params->n_batch, &n_past, &cur_pos_id, false);282 283 // generate the response284 285 LOG("\n");286 287 struct common_sampler * smpl = common_sampler_init(ctx_llava->model, params->sampling);288 if (!smpl) {289 LOG_ERR("%s: failed to initialize sampling subsystem\n", __func__);290 exit(1);291 }292 293 std::string response = "";294 for (int i = 0; i < max_tgt_len; i++) {295 const char * tmp = sample(smpl, ctx_llava->ctx_llama, &n_past, &cur_pos_id);296 response += tmp;297 if (strcmp(tmp, "</s>") == 0) break;298 if (strstr(tmp, "###")) break; // Yi-VL behavior299 LOG("%s", tmp);300 if (strstr(response.c_str(), "<|im_end|>")) break; // Yi-34B llava-1.6 - for some reason those decode not as the correct token (tokenizer works)301 if (strstr(response.c_str(), "<|im_start|>")) break; // Yi-34B llava-1.6302 if (strstr(response.c_str(), "USER:")) break; // mistral llava-1.6303 304 fflush(stdout);305 }306 307 common_sampler_free(smpl);308 LOG("\n");309}310 311static struct llama_model * llava_init(common_params * params) {312 llama_backend_init();313 llama_numa_init(params->numa);314 315 llama_model_params model_params = common_model_params_to_llama(*params);316 317 llama_model * model = llama_model_load_from_file(params->model.c_str(), model_params);318 if (model == NULL) {319 LOG_ERR("%s: unable to load model\n" , __func__);320 return NULL;321 }322 return model;323}324 325static struct llava_context * llava_init_context(common_params * params, llama_model * model) {326 const char * clip_path = params->mmproj.c_str();327 328 auto prompt = params->prompt;329 if (prompt.empty()) {330 prompt = "describe the image in detail.";331 }332 333 auto ctx_clip = clip_model_load(clip_path, /*verbosity=*/ 1);334 335 llama_context_params ctx_params = common_context_params_to_llama(*params);336 ctx_params.n_ctx = params->n_ctx < 2048 ? 2048 : params->n_ctx; // we need a longer context size to process image embeddings337 338 llama_context * ctx_llama = llama_init_from_model(model, ctx_params);339 340 if (ctx_llama == NULL) {341 LOG_ERR("%s: failed to create the llama_context\n" , __func__);342 return NULL;343 }344 345 auto * ctx_llava = (struct llava_context *)malloc(sizeof(llava_context));346 347 ctx_llava->ctx_llama = ctx_llama;348 ctx_llava->ctx_clip = ctx_clip;349 ctx_llava->model = model;350 return ctx_llava;351}352 353static void llava_free(struct llava_context * ctx_llava) {354 if (ctx_llava->ctx_clip) {355 clip_free(ctx_llava->ctx_clip);356 ctx_llava->ctx_clip = NULL;357 }358 359 llama_free(ctx_llava->ctx_llama);360 llama_model_free(ctx_llava->model);361 llama_backend_free();362}363 364#ifndef NDEBUG365 366static void debug_test_mrope_2d() {367 // 1. Initialize backend368 ggml_backend_t backend = NULL;369 std::string backend_name = "";370#ifdef GGML_USE_CUDA371 fprintf(stderr, "%s: using CUDA backend\n", __func__);372 backend = ggml_backend_cuda_init(0); // init device 0373 backend_name = "cuda";374 if (!backend) {375 fprintf(stderr, "%s: ggml_backend_cuda_init() failed\n", __func__);376 }377#endif378 // if there aren't GPU Backends fallback to CPU backend379 if (!backend) {380 backend = ggml_backend_cpu_init();381 backend_name = "cpu";382 }383 384 // Calculate the size needed to allocate385 size_t ctx_size = 0;386 ctx_size += 2 * ggml_tensor_overhead(); // tensors387 // no need to allocate anything else!388 389 // 2. Allocate `ggml_context` to store tensor data390 struct ggml_init_params params = {391 /*.mem_size =*/ ctx_size,392 /*.mem_buffer =*/ NULL,393 /*.no_alloc =*/ true, // the tensors will be allocated later by ggml_backend_alloc_ctx_tensors()394 };395 struct ggml_context * ctx = ggml_init(params);396 397 struct ggml_tensor * inp_raw = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, 128, 12, 30);398 ggml_set_name(inp_raw, "inp_raw");399 ggml_set_input(inp_raw);400 401 struct ggml_tensor * pos = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 30 * 4);402 ggml_set_name(pos, "pos");403 ggml_set_input(pos);404 405 std::vector<float> dummy_q;406 dummy_q.resize(128 * 12 * 30);407 std::fill(dummy_q.begin(), dummy_q.end(), 0.1);408 // memcpy(inp_raw->data, dummy_q.data(), 128 * 12 * 30 * ggml_element_size(inp_raw));409 410 std::vector<int> pos_id;411 pos_id.resize(30 * 4);412 for (int i = 0; i < 30; i ++) {413 pos_id[i] = i;414 pos_id[i + 30] = i + 10;415 pos_id[i + 60] = i + 20;416 pos_id[i + 90] = i + 30;417 }418 int sections[4] = {32, 32, 0, 0};419 420 // 4. Allocate a `ggml_backend_buffer` to store all tensors421 ggml_backend_buffer_t buffer = ggml_backend_alloc_ctx_tensors(ctx, backend);422 423 // 5. Copy tensor data from main memory (RAM) to backend buffer424 ggml_backend_tensor_set(inp_raw, dummy_q.data(), 0, ggml_nbytes(inp_raw));425 ggml_backend_tensor_set(pos, pos_id.data(), 0, ggml_nbytes(pos));426 427 // 6. Create a `ggml_cgraph` for mul_mat operation428 struct ggml_cgraph * gf = NULL;429 struct ggml_context * ctx_cgraph = NULL;430 431 // create a temporally context to build the graph432 struct ggml_init_params params0 = {433 /*.mem_size =*/ ggml_tensor_overhead()*GGML_DEFAULT_GRAPH_SIZE + ggml_graph_overhead(),434 /*.mem_buffer =*/ NULL,435 /*.no_alloc =*/ true, // the tensors will be allocated later by ggml_gallocr_alloc_graph()436 };437 ctx_cgraph = ggml_init(params0);438 gf = ggml_new_graph(ctx_cgraph);439 440 struct ggml_tensor * result0 = ggml_rope_multi(441 ctx_cgraph, inp_raw, pos, nullptr,442 128/2, sections, LLAMA_ROPE_TYPE_VISION, 32768, 1000000, 1,443 0, 1, 32, 1);444 445 // Add "result" tensor and all of its dependencies to the cgraph446 ggml_build_forward_expand(gf, result0);447 448 // 7. Create a `ggml_gallocr` for cgraph computation449 ggml_gallocr_t allocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend));450 ggml_gallocr_alloc_graph(allocr, gf);451 452 // 9. Run the computation453 int n_threads = 1; // Optional: number of threads to perform some operations with multi-threading454 if (ggml_backend_is_cpu(backend)) {455 ggml_backend_cpu_set_n_threads(backend, n_threads);456 }457 ggml_backend_graph_compute(backend, gf);458 459 // 10. Retrieve results (output tensors)460 // in this example, output tensor is always the last tensor in the graph461 struct ggml_tensor * result = result0;462 // struct ggml_tensor * result = gf->nodes[gf->n_nodes - 1];463 float * result_data = (float *)malloc(ggml_nbytes(result));464 // because the tensor data is stored in device buffer, we need to copy it back to RAM465 ggml_backend_tensor_get(result, result_data, 0, ggml_nbytes(result));466 const std::string bin_file = "mrope_2d_" + backend_name +".bin";467 std::ofstream outFile(bin_file, std::ios::binary);468 469 if (outFile.is_open()) {470 outFile.write(reinterpret_cast<const char*>(result_data), ggml_nbytes(result));471 outFile.close();472 std::cout << "Data successfully written to " + bin_file << std::endl;473 } else {474 std::cerr << "Error opening file!" << std::endl;475 }476 477 free(result_data);478 // 11. Free memory and exit479 ggml_free(ctx_cgraph);480 ggml_gallocr_free(allocr);481 ggml_free(ctx);482 ggml_backend_buffer_free(buffer);483 ggml_backend_free(backend);484}485 486static void debug_dump_img_embed(struct llava_context * ctx_llava) {487 int n_embd = llama_model_n_embd(llama_get_model(ctx_llava->ctx_llama));488 int ne = n_embd * 4;489 float vals[56 * 56 * 3];490 // float embd[ne];491 std::vector<float> embd;492 embd.resize(ne);493 494 for (int i = 0; i < 56*56; i++)495 {496 for (int c = 0; c < 3; c++)497 vals[i * 3 + c] = (float)(i % (56 * 56)) / (56*56);498 }499 500 clip_encode_float_image(ctx_llava->ctx_clip, 16, vals, 56, 56, embd.data());501 502 std::ofstream outFile("img_embed.bin", std::ios::binary);503 if (outFile.is_open()) {504 outFile.write(reinterpret_cast<const char*>(embd.data()), ne * sizeof(float));505 506 outFile.close();507 std::cout << "Data successfully written to mrope.bin" << std::endl;508 } else {509 std::cerr << "Error opening file!" << std::endl;510 }511}512 513#endif514 515 516int main(int argc, char ** argv) {517 ggml_time_init();518 519 common_params params;520 521 if (!common_params_parse(argc, argv, params, LLAMA_EXAMPLE_LLAVA, print_usage)) {522 return 1;523 }524 525 common_init();526 527 if (params.mmproj.empty() || (params.image.empty() && !prompt_contains_image(params.prompt))) {528 print_usage(argc, argv);529 return 1;530 }531 532 auto * model = llava_init(¶ms);533 if (model == NULL) {534 fprintf(stderr, "%s: error: failed to init llava model\n", __func__);535 return 1;536 }537 538 if (prompt_contains_image(params.prompt)) {539 auto * ctx_llava = llava_init_context(¶ms, model);540 541 auto * image_embed = load_image(ctx_llava, ¶ms, "");542 543 // process the prompt544 process_prompt(ctx_llava, image_embed, ¶ms, params.prompt);545 546 llama_perf_context_print(ctx_llava->ctx_llama);547 llava_image_embed_free(image_embed);548 ctx_llava->model = NULL;549 llava_free(ctx_llava);550#ifndef NDEBUG551 } else if (params.image[0].empty()) {552 auto ctx_llava = llava_init_context(¶ms, model);553 554 debug_test_mrope_2d();555 debug_dump_img_embed(ctx_llava);556 557 llama_perf_context_print(ctx_llava->ctx_llama);558 ctx_llava->model = NULL;559 llava_free(ctx_llava);560#endif561 } else {562 for (auto & image : params.image) {563 auto * ctx_llava = llava_init_context(¶ms, model);564 565 auto * image_embed = load_image(ctx_llava, ¶ms, image);566 if (!image_embed) {567 LOG_ERR("%s: failed to load image %s. Terminating\n\n", __func__, image.c_str());568 return 1;569 }570 571 // process the prompt572 process_prompt(ctx_llava, image_embed, ¶ms, params.prompt);573 574 llama_perf_context_print(ctx_llava->ctx_llama);575 llava_image_embed_free(image_embed);576 ctx_llava->model = NULL;577 llava_free(ctx_llava);578 }579 }580 581 llama_model_free(model);582 583 return 0;584}585