Brunobkr/llama.cpp_AlgMor24_github

1#include "models.h"2 3ggml_cgraph * clip_graph_deepseekocr2::build() {4    GGML_ASSERT(hparams.n_head_kv > 0);5    GGML_ASSERT(n_head % hparams.n_head_kv == 0);6 7    // patch embedding8    ggml_tensor * inp_raw = build_inp_raw();9 10    ggml_tensor * sam_out = build_sam(inp_raw);11 12    ggml_tensor * qwen2_out;13    // Building Qwen2 encoder14    {15        ggml_tensor * inp;16 17        // H*W, C, B18        inp = ggml_reshape_3d(ctx0, sam_out, sam_out->ne[0] * sam_out->ne[1], sam_out->ne[2], sam_out->ne[3]);19        inp = ggml_cont(ctx0, ggml_permute(ctx0, inp, 1, 0, 2, 3)); // C, H*W, B20 21        auto num_image_tokens = inp->ne[1]; // H*W22        GGML_ASSERT(num_image_tokens == 144 || num_image_tokens == 256);23 24        // query based on numbers of image tokens (in SAM output)25        // 16x16 -> query_1024 (1024x1024 images)26        // 12x12 -> query_768 (768x768 images)27 28        ggml_tensor * query_embed = model.resample_query_1024;29        int           num_queries = 256;30 31        if (num_image_tokens == 144) {32            query_embed = model.resample_query_768;33            num_queries = 144;34        }35 36        // repeat the query embedding per batch item, then append: (C, num_image_tokens + num_queries, B)37        query_embed = ggml_cast(ctx0, query_embed, inp->type);38        query_embed = ggml_repeat_4d(ctx0, query_embed, query_embed->ne[0], num_queries, inp->ne[2], 1);39        inp = ggml_concat(ctx0, inp, query_embed, 1);40 41        auto seq_len = inp->ne[1];42 43        // qwen2 encoder attention mask44        ggml_tensor * attn_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, seq_len, seq_len);45        ggml_set_name(attn_mask, "qwen2_attn_mask");46        ggml_set_input(attn_mask);47 48        ggml_tensor * inp_pos = ggml_cast(ctx0, ggml_arange(ctx0, 0, seq_len, 1), GGML_TYPE_I32);49 50        auto add_rope = [&](ggml_tensor * x, const clip_layer &) {51            return ggml_rope_ext(ctx0, x, inp_pos, nullptr, d_head,52                                 GGML_ROPE_TYPE_NEOX, 131072, 1000000, 1, 0, 1, 0, 0);53        };54 55        build_vit_opts vit_opts;56        vit_opts.attn_mask = attn_mask;57 58        // build_vit applies model.post_ln_w internally; do not re-apply59        ggml_tensor * cur = build_vit(inp, seq_len, NORM_TYPE_RMS, FFN_SILU,60                                      /* learned_pos_embd */ nullptr, add_rope, vit_opts);61 62        cur = ggml_cont(ctx0,63                        ggml_view_3d(ctx0, cur, cur->ne[0], num_queries, cur->ne[2], cur->nb[1], cur->nb[2],64                                     cur->nb[1] * (cur->ne[1] - num_queries))); // only take query tokens for output65 66        ggml_build_forward_expand(gf, cur);67        qwen2_out = cur;68    }69 70    ggml_tensor * cur;71 72    cur = ggml_mul_mat(ctx0, model.mm_fc_w, qwen2_out);73    cur = ggml_add(ctx0, cur, model.mm_fc_b);74 75    // view_seperator only after the global view76    if (img.add_viewsep) {77        ggml_tensor * vs = ggml_repeat_4d(ctx0, model.view_seperator, model.view_seperator->ne[0], 1, cur->ne[2], 1);78        cur = ggml_concat(ctx0, cur, vs, 1); // (n_dim, 257, n_batch)79    }80 81    cb(cur, "dsocr2_output", -1);82 83    ggml_build_forward_expand(gf, cur);84    return gf;85}86