KBaba7/llama.cpp

1#include "clip.h"2#include "llava.h"3 4#include "llama.h"5 6#include <algorithm>7#include <cerrno>8#include <cstdio>9#include <cstdlib>10#include <cstring>11#include <limits>12#include <vector>13 14#if defined(LLAVA_LOG_OFF)15#   define LOG_INF(...)16#   define LOG_WRN(...)17#   define LOG_ERR(...)18#   define LOG_DBG(...)19#else // defined(LLAVA_LOG_OFF)20#   define LOG_INF(...) do { fprintf(stdout, __VA_ARGS__); } while (0)21#   define LOG_WRN(...) do { fprintf(stderr, __VA_ARGS__); } while (0)22#   define LOG_ERR(...) do { fprintf(stderr, __VA_ARGS__); } while (0)23#   define LOG_DBG(...) do { fprintf(stdout, __VA_ARGS__); } while (0)24#endif // defined(LLAVA_LOG_OFF)25 26// RGB uint8 image27struct clip_image_u8 {28    int nx;29    int ny;30 31    std::vector<uint8_t> buf;32};33 34// RGB float32 image (NHWC)35// Memory layout: RGBRGBRGB...36struct clip_image_f32 {37    int nx;38    int ny;39 40    std::vector<float> buf;41};42 43struct clip_image_grid_shape {44    int first;45    int second;46};47 48/**49 * Selects the best resolution from a list of possible resolutions based on the original size.50 *51 * @param original_size The original size of the image in the format (width, height).52 * @param possible_resolutions A list of possible resolutions in the format [(width1, height1), (width2, height2), ...].53 * @return The best fit resolution in the format (width, height).54 */55static std::pair<int, int> select_best_resolution(const std::pair<int, int>& original_size, const std::vector<std::pair<int, int>>& possible_resolutions) {56    int original_width  = original_size.first;57    int original_height = original_size.second;58 59    std::pair<int, int> best_fit;60    int max_effective_resolution = 0;61    int min_wasted_resolution = std::numeric_limits<int>::max();62 63    for (const auto& resolution : possible_resolutions) {64        int width = resolution.first;65        int height = resolution.second;66        float scale = std::min(static_cast<float>(width) / original_width, static_cast<float>(height) / original_height);67        int downscaled_width  = static_cast<int>(original_width * scale);68        int downscaled_height = static_cast<int>(original_height * scale);69        int effective_resolution = std::min(downscaled_width * downscaled_height, original_width * original_height);70        int wasted_resolution = (width * height) - effective_resolution;71        // LOG_DBG("resolution: %d %d, scale: %f, downscaled: %d %d, effective: %d, wasted: %d\n", width, height, scale, downscaled_width, downscaled_height, effective_resolution, wasted_resolution);72        if (effective_resolution > max_effective_resolution || (effective_resolution == max_effective_resolution && wasted_resolution < min_wasted_resolution)) {73            max_effective_resolution = effective_resolution;74            min_wasted_resolution = wasted_resolution;75            best_fit = resolution;76        }77    }78 79    return best_fit;80}81 82/**83 * @brief Get the anyres image grid shape object84 *85 * @param image_size86 * @param grid_pinpoints87 * @param image_patch_size88 * @return <int, int>89 */90static struct clip_image_grid_shape get_anyres_image_grid_shape(const std::pair<int, int> & image_size, const std::vector<std::pair<int, int>> & grid_pinpoints, int image_patch_size) {91    /**92        Conversion from gguf flat array to vector:93        std::vector<std::pair<int, int>> possible_resolutions;94        for (int i = 0; i < 32 && params.image_grid_pinpoints[i] != 0; i+=2) {95            possible_resolutions.push_back({params.image_grid_pinpoints[i], params.image_grid_pinpoints[i+1]});96        }97     */98    auto best_resolution = select_best_resolution(image_size, grid_pinpoints);99    return {best_resolution.first / image_patch_size, best_resolution.second / image_patch_size};100}101 102// Take the image segments in a grid configuration and return the embeddings and the number of embeddings into preallocated memory (image_embd_out)103static bool clip_llava_handle_patches(clip_ctx * ctx_clip, std::vector<float *> & image_embd_v, struct clip_image_grid_shape grid_shape, float * image_embd_out, int * n_img_pos_out) {104    struct {105        struct ggml_context * ctx;106    } model;107 108    const int32_t image_size = clip_image_size(ctx_clip);109    const int32_t patch_size = clip_patch_size(ctx_clip);110 111    int32_t num_patches_per_side = image_size / patch_size; // 336 / 14 = 24 - used for embedding-patching boxes (24*24 = 576 patches)112 113    int num_patches_width  = grid_shape.first;  // grid 1-4114    int num_patches_height = grid_shape.second; // grid 1-4115 116    const size_t num_images = num_patches_width * num_patches_height + 1;117 118    // TODO: size calculation is not calculated - it's only tens of MB119    size_t ctx_size = 0;120 121    {122        ctx_size += clip_embd_nbytes(ctx_clip) * num_images * 8; // image_features123        ctx_size += 1024*1024 * ggml_type_size(GGML_TYPE_F32);124    }125 126    struct ggml_init_params params {127        /*.mem_size   =*/ ctx_size,128        /*.mem_buffer =*/ NULL,129        /*.no_alloc   =*/ false, // NOTE: this should be false when using the legacy API130    };131 132    // Python reference code for full unpad:133    /*134        base_image_feature = image_feature[0]135        image_feature = image_feature[1:]136        image_feature = image_feature.permute(4, 0, 2, 1, 3).contiguous()137        image_feature = image_feature.flatten(1, 2).flatten(2, 3)138        image_feature = unpad_image(image_feature, image_sizes[image_idx])139        image_feature = torch.cat((140            image_feature,141            self.model.image_newline[:, None, None].expand(*image_feature.shape[:-1], 1)142        ), dim=-1)143        image_feature = image_feature.flatten(1, 2).transpose(0, 1)144        image_feature = torch.cat((base_image_feature, image_feature), dim=0)145    */146    // We now have two options: unpad or no unpad. Unpad removes tokens for faster llm eval.147    // In terms of result quality it appears to make no difference, so we'll start with the easier approach given 5D tensors are not supported in ggml yet.148    // Without unpad we have to split the sub-image embeddings into patches of 24 features each and permute them.149    // Once all images are processed to prepended the base_image_features without any changes.150 151    // Pytorch reference simplified, modified for ggml compatibility - confirmed identical output in python (for a 2x2 grid image (676x676 scaling))152    /*153        image_feature = image_feature.view(2, 2, 24, 24, 4096)154        image_feature = image_feature.permute(0, 2, 1, 3, 4).contiguous()155        image_feature = image_feature.view(2, 24, 2, 24, 4096)156        image_feature = image_feature.flatten(0, 3)157 158        // Reshape to 4D tensor by merging the last two dimensions159        image_feature = image_feature.view(2, 2, 24, 24*4096)160        image_feature = image_feature.permute(0, 2, 1, 3).contiguous()161        image_feature = image_feature.view(-1, 4096)162    */163 164    model.ctx = ggml_init(params);165 166    struct ggml_tensor * image_features = ggml_new_tensor_3d(model.ctx, GGML_TYPE_F32, clip_n_mmproj_embd(ctx_clip), clip_n_patches(ctx_clip), num_images - 1); // example: 4096 x 576 x 4167    // ggml_tensor_printf(image_features,"image_features",__LINE__,false,false);168    // fill it with the image embeddings, ignoring the base169    for (size_t i = 1; i < num_images; i++) {170        size_t offset = (i-1) * clip_embd_nbytes(ctx_clip);171        memcpy((uint8_t *)(image_features->data) + offset, image_embd_v[i], clip_embd_nbytes(ctx_clip));172    }173 174    struct ggml_cgraph  * gf = ggml_new_graph(model.ctx);175    size_t size_ele = ggml_type_size(GGML_TYPE_F32);176 177    struct ggml_tensor *image_features_patchview = ggml_view_4d(model.ctx, image_features,178                                                                num_patches_per_side * clip_n_mmproj_embd(ctx_clip),179                                                                num_patches_per_side,180                                                                num_patches_width,181                                                                num_patches_height,182                                                                size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip),183                                                                size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip) * num_patches_per_side,184                                                                size_ele * num_patches_per_side * clip_n_mmproj_embd(ctx_clip) * num_patches_per_side * num_patches_width, 0);185    // ggml_tensor_printf(image_features_patchview,"image_features_patchview",__LINE__,false,false);186    struct ggml_tensor *permuted_cont = ggml_cont(model.ctx, ggml_permute(model.ctx, image_features_patchview, 0, 2, 1, 3));187    /**188     At the end of each row we have to add the row_end embeddings, which are the same as the newline embeddings189         image_feature = torch.cat((190        image_feature,191        self.model.image_newline[:, None, None].expand(*image_feature.shape[:-1], 1).to(image_feature.device)192    ), dim=-1)193     *194     */195 196    // ggml_tensor_printf(permuted_cont,"permuted_cont",__LINE__,false,false);197    struct ggml_tensor *flatten = ggml_view_2d(model.ctx, permuted_cont, clip_n_mmproj_embd(ctx_clip), num_patches_height * num_patches_width * num_patches_per_side * num_patches_per_side,  size_ele * clip_n_mmproj_embd(ctx_clip), 0);198    // ggml_tensor_printf(flatten,"flatten",__LINE__,false,false);199    ggml_build_forward_expand(gf, flatten);200    ggml_graph_compute_with_ctx(model.ctx, gf, 1);201    struct ggml_tensor* result = ggml_graph_node(gf, -1);202 203    memcpy(image_embd_out, image_embd_v[0], clip_embd_nbytes(ctx_clip)); // main image as global context204    // append without newline tokens (default behavior in llava_arch when not using unpad ):205    memcpy(image_embd_out + clip_n_patches(ctx_clip) * clip_n_mmproj_embd(ctx_clip), (float*)result->data, clip_embd_nbytes(ctx_clip) * (num_images-1)); // grid patches206    *n_img_pos_out = static_cast<int>(result->ne[1]+clip_n_patches(ctx_clip));207 208    // Debug: Test single segments209    // Current findings: sending base image, sending a segment embedding all works similar to python210    // However, permuted embeddings do not work yet (stride issue?)211    // memcpy(image_embd_out, image_embd_v[0], clip_embd_nbytes(ctx_clip)); // main image as context212    // memcpy(image_embd_out, (float*)prepared_cont->data, clip_embd_nbytes(ctx_clip)); // main image as context213    // *n_img_pos_out=576;214 215    ggml_free(model.ctx);216    return true;217}218 219static clip_image_f32 * reshape_by_patch(clip_image_f32 * image, int patch_size) {220    int width = image->nx;221    int height = image->ny;222    int num_patches = (height / patch_size) * (width / patch_size);223    clip_image_f32 * patch = clip_image_f32_init();224    patch->nx = patch_size * num_patches;225    patch->ny = patch_size;226    patch->buf.resize(3 * patch->nx * patch->ny);227 228    int patch_index = 0;229 230    for (int i = 0; i < height; i += patch_size) {231        for (int j = 0; j < width; j += patch_size) {232            for (int pi = 0; pi < patch_size; ++pi) {233                for (int pj = 0; pj < patch_size; ++pj) {234                    int input_index = ((i + pi) * width + (j + pj)) * 3;235                    int output_index = (pi * patch_size * num_patches + patch_index * patch_size + pj) * 3;236                    patch->buf[output_index] = image->buf[input_index];237                    patch->buf[output_index+1] = image->buf[input_index+1];238                    patch->buf[output_index+2] = image->buf[input_index+2];239                }240            }241            patch_index++;242        }243    }244    return patch;245}246 247static bool encode_image_with_clip(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float * image_embd, int * n_img_pos) {248    // std::vector<clip_image_f32*> img_res_v; // format VectN x H x W x RGB (N x 336 x 336 x 3), so interleaved RGB - different to the python implementation which is N x 3 x 336 x 336249    clip_image_f32_batch img_res_v;250    img_res_v.size = 0;251    img_res_v.data = nullptr;252    if (!clip_image_preprocess(ctx_clip, img, &img_res_v)) {253        LOG_ERR("%s: unable to preprocess image\n", __func__);254        delete[] img_res_v.data;255        return false;256    }257 258    const int64_t t_img_enc_start_us = ggml_time_us();259 260    const char * mm_patch_merge_type = clip_patch_merge_type(ctx_clip);261 262    if (clip_is_minicpmv(ctx_clip) || clip_is_qwen2vl(ctx_clip)) {263        std::vector<float *> image_embd_v;264        image_embd_v.resize(img_res_v.size);265        struct clip_image_size * load_image_size = clip_image_size_init();266 267        for (size_t i = 0; i < img_res_v.size; i++) {268            const int64_t t_img_enc_step_start_us = ggml_time_us();269            image_embd_v[i] = (float *)malloc(clip_embd_nbytes_by_img(ctx_clip, img_res_v.data[i].nx, img_res_v.data[i].ny));270            int patch_size=14;271            load_image_size->width = img_res_v.data[i].nx;272            load_image_size->height = img_res_v.data[i].ny;273            clip_add_load_image_size(ctx_clip, load_image_size);274 275            bool encoded = false;276            if (clip_is_qwen2vl(ctx_clip)) {277                encoded = clip_image_encode(ctx_clip, n_threads, &img_res_v.data[i], image_embd_v[i]);278            }279            else {280                encoded = clip_image_encode(ctx_clip, n_threads, reshape_by_patch(&img_res_v.data[i], patch_size), image_embd_v[i]);281            }282 283            if (!encoded) {284                LOG_ERR("Unable to encode image - spatial_unpad - subimage %d of %d\n", (int) i+1, (int) img_res_v.size);285                return false;286            }287            const int64_t t_img_enc_steop_batch_us = ggml_time_us();288            LOG_INF("%s: step %d of %d encoded in %8.2f ms\n", __func__, (int)i+1, (int)img_res_v.size, (t_img_enc_steop_batch_us - t_img_enc_step_start_us) / 1000.0);289        }290        const int64_t t_img_enc_batch_us = ggml_time_us();291        LOG_INF("%s: all %d segments encoded in %8.2f ms\n", __func__, (int)img_res_v.size, (t_img_enc_batch_us - t_img_enc_start_us) / 1000.0);292 293        int n_img_pos_out = 0;294        for (size_t i = 0; i < image_embd_v.size(); i++) {295            std::memcpy(296                image_embd + n_img_pos_out * clip_n_mmproj_embd(ctx_clip),297                image_embd_v[i],298                clip_embd_nbytes_by_img(ctx_clip, img_res_v.data[i].nx, img_res_v.data[i].ny));299            n_img_pos_out += clip_n_patches_by_img(ctx_clip, &img_res_v.data[i]);300        }301        *n_img_pos = n_img_pos_out;302        for (size_t i = 0; i < image_embd_v.size(); i++) {303            free(image_embd_v[i]);304        }305        image_embd_v.clear();306        load_image_size->width = img->nx;307        load_image_size->height = img->ny;308        clip_add_load_image_size(ctx_clip, load_image_size);309        LOG_INF("%s: load_image_size %d %d\n", __func__, load_image_size->width, load_image_size->height);310        delete[] img_res_v.data;311        img_res_v.size = 0;312        img_res_v.data = nullptr;313    }314    else if (clip_is_glm(ctx_clip)){315        struct clip_image_size * load_image_size = clip_image_size_init();316        load_image_size->width = img_res_v.data[0].nx;317        load_image_size->height = img_res_v.data[0].ny;318        clip_add_load_image_size(ctx_clip, load_image_size);319 320        bool encoded = clip_image_encode(ctx_clip, n_threads, &img_res_v.data[0], image_embd);321        int pos = int(load_image_size->width/clip_patch_size(ctx_clip)/2);322        *n_img_pos = (pos * pos + 2);323        if (!encoded){324            LOG_ERR("Unable to encode image \n");325            return false;326        }327    }328    else if (strcmp(mm_patch_merge_type, "spatial_unpad") != 0) {329        // flat / default llava-1.5 type embedding330        *n_img_pos = clip_n_patches(ctx_clip);331        bool encoded = clip_image_encode(ctx_clip, n_threads, &img_res_v.data[0], image_embd); // image_embd shape is 576 x 4096332        delete[] img_res_v.data;333        if (!encoded) {334            LOG_ERR("Unable to encode image\n");335 336            return false;337        }338    }339    else {340        // spatial_unpad llava-1.6 type embedding341        // TODO: CLIP needs batching support - in HF the llm projection is separate after encoding, which might be a solution to quickly get batching working342        std::vector<float *> image_embd_v;343        image_embd_v.resize(img_res_v.size);344        for (size_t i = 0; i < img_res_v.size; i++) {345            image_embd_v[i] = (float *)malloc(clip_embd_nbytes(ctx_clip)); // 576 patches * 4096 embeddings * 4 bytes = 9437184346            const bool encoded = clip_image_encode(ctx_clip, n_threads, &img_res_v.data[i], image_embd_v[i]); // image data is in 3x336x336 format and will be converted to 336x336x3 inside347            if (!encoded) {348                LOG_ERR("Unable to encode image - spatial_unpad - subimage %d of %d\n", (int) i+1, (int) img_res_v.size);349                return false;350            }351        }352        const int64_t t_img_enc_batch_us = ggml_time_us();353        LOG_INF("%s: %d segments encoded in %8.2f ms\n", __func__, (int)img_res_v.size, (t_img_enc_batch_us - t_img_enc_start_us) / 1000.0);354 355        const int32_t * image_grid = clip_image_grid(ctx_clip);356 357        std::vector<std::pair<int, int>> grid_pinpoints;358        for (int i = 0; i < 32 && image_grid[i] != 0; i += 2) {359            grid_pinpoints.push_back({image_grid[i], image_grid[i+1]});360        }361 362        // free all img_res_v - not needed anymore363        delete[] img_res_v.data;364        img_res_v.size = 0;365        img_res_v.data = nullptr;366 367        const int32_t image_size = clip_image_size(ctx_clip);368 369        struct clip_image_grid_shape grid_shape = get_anyres_image_grid_shape({img->nx,img->ny}, grid_pinpoints, image_size);370 371        int n_img_pos_out;372        clip_llava_handle_patches(ctx_clip, image_embd_v, grid_shape, image_embd, &n_img_pos_out);373        *n_img_pos = n_img_pos_out;374 375        for (size_t i = 0; i < image_embd_v.size(); i++) {376            free(image_embd_v[i]);377        }378        image_embd_v.clear();379 380        // debug image/segment/normalization content:381        // clip_image_u8 * tmp = clip_image_u8_init();382        // clip_image_convert_f32_to_u8(*image_feature, *tmp);383        // clip_image_save_to_bmp(*tmp, "image_feature.bmp");384    }385 386    LOG_INF("%s: image embedding created: %d tokens\n", __func__, *n_img_pos);387 388    const int64_t t_img_enc_end_us = ggml_time_us();389    float t_img_enc_ms = (t_img_enc_end_us - t_img_enc_start_us) / 1000.0;390 391    LOG_INF("\n%s: image encoded in %8.2f ms by CLIP (%8.2f ms per image patch)\n", __func__, t_img_enc_ms, t_img_enc_ms / *n_img_pos);392 393    return true;394}395 396bool llava_validate_embed_size(const llama_context * ctx_llama, const clip_ctx * ctx_clip) {397        // make sure that the correct mmproj was used, i.e., compare apples to apples398    int n_llama_embd = llama_model_n_embd(llama_get_model(ctx_llama));399    auto n_image_embd = clip_n_mmproj_embd(ctx_clip);400    if (n_image_embd != n_llama_embd) {401        LOG_ERR("%s: embedding dim of the multimodal projector (%d) is not equal to that of LLaMA (%d). Make sure that you use the correct mmproj file.\n", __func__, n_image_embd, n_llama_embd);402        return false;403    }404    return true;405}406 407bool llava_image_embed_make_with_clip_img(clip_ctx * ctx_clip, int n_threads, const clip_image_u8 * img, float ** image_embd_out, int * n_img_pos_out) {408    int num_max_patches = 6;409    if (clip_is_minicpmv(ctx_clip)) {410        num_max_patches = 10;411    }412    if (clip_is_glm(ctx_clip)) {413        num_max_patches = 1;414    }415    float * image_embd;416    if (clip_is_qwen2vl(ctx_clip)) {417        // qwen2vl don't split image into chunks, so `num_max_patches` is not needed.418        image_embd = (float *)malloc(clip_embd_nbytes_by_img(ctx_clip, img->nx, img->ny));419    } else {420        image_embd = (float *)malloc(clip_embd_nbytes(ctx_clip)*num_max_patches); // TODO: base on gridsize/llava model421    }422    if (!image_embd) {423        LOG_ERR("Unable to allocate memory for image embeddings\n");424        return false;425    }426 427    int n_img_pos;428    if (!encode_image_with_clip(ctx_clip, n_threads, img, image_embd, &n_img_pos)) {429        LOG_ERR("%s: cannot encode image, aborting\n", __func__);430        free(image_embd);431        return false;432    }433    *image_embd_out = image_embd;434    *n_img_pos_out = n_img_pos;435 436    return true;437}438 439struct llava_embd_batch {440    std::vector<llama_pos>      pos;441    std::vector<int32_t>        n_seq_id;442    std::vector<llama_seq_id>   seq_id_0;443    std::vector<llama_seq_id *> seq_ids;444    std::vector<int8_t>         logits;445    llama_batch batch;446    llava_embd_batch(float * embd, int32_t n_tokens, llama_pos pos_0, llama_seq_id seq_id) {447        pos     .resize(n_tokens);448        n_seq_id.resize(n_tokens);449        seq_ids .resize(n_tokens + 1);450        logits  .resize(n_tokens);451        seq_id_0.resize(1);452        seq_id_0[0] = seq_id;453        seq_ids [n_tokens] = nullptr;454        batch = {455            /*n_tokens       =*/ n_tokens,456            /*tokens         =*/ nullptr,457            /*embd           =*/ embd,458            /*pos            =*/ pos.data(),459            /*n_seq_id       =*/ n_seq_id.data(),460            /*seq_id         =*/ seq_ids.data(),461            /*logits         =*/ logits.data(),462        };463        for (int i = 0; i < n_tokens; i++) {464            batch.pos     [i] = pos_0 + i;465            batch.n_seq_id[i] = 1;466            batch.seq_id  [i] = seq_id_0.data();467            batch.logits  [i] = false;468        }469    }470};471 472bool llava_eval_image_embed(llama_context * ctx_llama, const struct llava_image_embed * image_embed, int n_batch, int * n_past) {473    int n_embd  = llama_model_n_embd(llama_get_model(ctx_llama));474 475    for (int i = 0; i < image_embed->n_image_pos; i += n_batch) {476        int n_eval = image_embed->n_image_pos - i;477        if (n_eval > n_batch) {478            n_eval = n_batch;479        }480        float * embd = image_embed->embed+i*n_embd;481        llava_embd_batch llava_batch = llava_embd_batch(embd, n_eval, *n_past, 0);482        if (llama_decode(ctx_llama, llava_batch.batch)) {483            LOG_ERR("%s : failed to eval\n", __func__);484            return false;485        }486        *n_past += n_eval;487    }488    return true;489}490 491struct llava_image_embed * llava_image_embed_make_with_bytes(struct clip_ctx * ctx_clip, int n_threads, const unsigned char * image_bytes, int image_bytes_length) {492    clip_image_u8 * img = clip_image_u8_init();493    if (!clip_image_load_from_bytes(image_bytes, image_bytes_length, img)) {494        clip_image_u8_free(img);495        LOG_ERR("%s: can't load image from bytes, is it a valid image?", __func__);496        return NULL;497    }498 499    float* image_embed = NULL;500    int n_image_pos = 0;501    bool image_embed_result = llava_image_embed_make_with_clip_img(ctx_clip, n_threads, img, &image_embed, &n_image_pos);502    if (!image_embed_result) {503        clip_image_u8_free(img);504        LOG_ERR("%s: couldn't embed the image\n", __func__);505        return NULL;506    }507 508    clip_image_u8_free(img);509    auto result = (llava_image_embed*)malloc(sizeof(llava_image_embed));510    result->embed = image_embed;511    result->n_image_pos = n_image_pos;512    return result;513}514 515static bool load_file_to_bytes(const char* path, unsigned char** bytesOut, long *sizeOut) {516    auto file = fopen(path, "rb");517    if (file == NULL) {518        LOG_ERR("%s: can't read file %s\n", __func__, path);519        return false;520    }521 522    fseek(file, 0, SEEK_END);523    auto fileSize = ftell(file);524    fseek(file, 0, SEEK_SET);525 526    auto buffer = (unsigned char *)malloc(fileSize); // Allocate memory to hold the file data527    if (buffer == NULL) {528        LOG_ERR("%s: failed to alloc %ld bytes for file %s\n", __func__, fileSize, path);529        perror("Memory allocation error");530        fclose(file);531        return false;532    }533    errno = 0;534    size_t ret = fread(buffer, 1, fileSize, file); // Read the file into the buffer535    if (ferror(file)) {536        LOG_ERR("read error: %s", strerror(errno));537        free(buffer);538        fclose(file);539        return false;540    }541    if (ret != (size_t) fileSize) {542        LOG_ERR("unexpectedly reached end of file");543        free(buffer);544        fclose(file);545        return false;546    }547    fclose(file); // Close the file548 549    *bytesOut = buffer;550    *sizeOut = fileSize;551    return true;552}553 554struct llava_image_embed * llava_image_embed_make_with_filename(struct clip_ctx * ctx_clip, int n_threads, const char * image_path) {555    unsigned char* image_bytes;556    long image_bytes_length;557    auto loaded = load_file_to_bytes(image_path, &image_bytes, &image_bytes_length);558    if (!loaded) {559        LOG_ERR("%s: failed to load %s\n", __func__, image_path);560        return NULL;561    }562 563    llava_image_embed *embed = llava_image_embed_make_with_bytes(ctx_clip, n_threads, image_bytes, image_bytes_length);564    free(image_bytes);565 566    return embed;567}568 569void llava_image_embed_free(struct llava_image_embed * embed) {570    free(embed->embed);571    free(embed);572}573