Brunobkr/llama.cpp_AlgMor24_github

1#include "models.h"2 3ggml_cgraph * clip_graph_internvl::build() {4    GGML_ASSERT(model.class_embedding != nullptr);5    GGML_ASSERT(model.position_embeddings != nullptr);6 7    const int n_pos = n_patches + 1;8    ggml_tensor * inp = build_inp();9 10    // add CLS token11    ggml_tensor * cls_repeated = ggml_repeat_4d(ctx0, model.class_embedding,12            model.class_embedding->ne[0], 1, n_batch, 1);13    inp = ggml_concat(ctx0, inp, cls_repeated, 1);14 15    // The larger models use a different ViT, which uses RMS norm instead of layer norm16    // ref: https://github.com/ggml-org/llama.cpp/pull/13443#issuecomment-286978618817    norm_type norm_t = (hparams.n_embd == 3200 && hparams.n_layer == 45)18        ? NORM_TYPE_RMS // 6B ViT (Used by InternVL 2.5/3 - 26B, 38B, 78B)19        : NORM_TYPE_NORMAL; // 300M ViT (Used by all smaller InternVL models)20 21    ggml_tensor * cur = build_vit(22                            inp, n_pos,23                            norm_t,24                            hparams.ffn_op,25                            model.position_embeddings,26                            nullptr);27 28    // remove CLS token29    cur = ggml_view_3d(ctx0, cur,30        n_embd, n_patches, n_batch,31        cur->nb[1], cur->nb[2], 0);32    cur = ggml_cont(ctx0, cur);33 34    // pixel shuffle35    {36        const int scale_factor = model.hparams.n_merge;37        const int bsz    = n_batch;38        const int height = n_patches_y;39        const int width  = n_patches_x;40        GGML_ASSERT(scale_factor > 0);41        cur = ggml_reshape_4d(ctx0, cur, n_embd * scale_factor, height / scale_factor, width, bsz);42        cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);43        cur = ggml_cont_4d(ctx0, cur,44            n_embd * scale_factor * scale_factor,45            height / scale_factor,46            width / scale_factor,47            bsz);48        cur = ggml_permute(ctx0, cur, 0, 2, 1, 3);49        // flatten to 2D50        cur = ggml_cont_3d(ctx0, cur,51            n_embd * scale_factor * scale_factor,52            cur->ne[1] * cur->ne[2],53            cur->ne[3]);54    }55 56    // projector (always using GELU activation)57    {58        // projector LayerNorm uses pytorch's default eps = 1e-559        // ref: https://huggingface.co/OpenGVLab/InternVL3-8B-Instruct/blob/a34d3e4e129a5856abfd6aa6de79776484caa14e/modeling_internvl_chat.py#L7960        cur = build_norm(cur, model.mm_0_w, model.mm_0_b, NORM_TYPE_NORMAL, 1e-5, -1);61        cur = build_ffn(cur,62            model.mm_1_w, model.mm_1_b,63            nullptr, nullptr,64            model.mm_3_w, model.mm_3_b,65            FFN_GELU,66            -1);67    }68 69    // build the graph70    ggml_build_forward_expand(gf, cur);71 72    return gf;73}74