Brunobkr/llama.cpp_AlgMor24_github

1#include "models.h"2 3ggml_cgraph * clip_graph_qwen3vl::build() {4    GGML_ASSERT(model.patch_bias != nullptr);5    GGML_ASSERT(model.position_embeddings != nullptr);6    GGML_ASSERT(model.class_embedding == nullptr);7 8    const int batch_size       = 1;9    const int n_pos            = n_patches;10    const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position11 12    norm_type norm_t = NORM_TYPE_NORMAL;13 14    int mrope_sections[4] = {d_head/4, d_head/4, d_head/4, d_head/4};15 16    ggml_tensor * inp = build_inp_with_temporal_merge();17 18    // spatial merge19    {20        inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);  // [w, h, c, b] -> [c, w, h, b]21        inp = ggml_cont_4d(22            ctx0, inp,23            n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);24        inp = ggml_reshape_4d(25            ctx0, inp,26            n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));27        inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);28        inp = ggml_cont_3d(29            ctx0, inp,30            n_embd, n_patches_x * n_patches_y, batch_size);31    }32 33    // add patch bias34    if (model.patch_bias != nullptr) {35        inp = ggml_add(ctx0, inp, model.patch_bias);36        cb(inp, "patch_bias", -1);37    }38 39    // calculate absolute position embedding and apply40    ggml_tensor * learned_pos_embd = resize_position_embeddings(GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS);41    learned_pos_embd = ggml_cont_4d(42        ctx0, learned_pos_embd,43        n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);44    learned_pos_embd = ggml_reshape_4d(45        ctx0, learned_pos_embd,46        n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));47    learned_pos_embd = ggml_permute(ctx0, learned_pos_embd, 0, 2, 1, 3);48    learned_pos_embd = ggml_cont_3d(49        ctx0, learned_pos_embd,50        n_embd, n_patches_x * n_patches_y, batch_size);51    inp = ggml_add(ctx0, inp, learned_pos_embd);52    cb(inp, "inp_pos_emb", -1);53 54    ggml_tensor * inpL = inp;55 56    ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);57    ggml_set_name(positions, "positions");58    ggml_set_input(positions);59 60    // pre-layernorm61    if (model.pre_ln_w) {62        inpL = build_norm(inpL, model.pre_ln_w, model.pre_ln_b, norm_t, eps, -1);63    }64 65    // deepstack features (stack along the feature dimension), [n_embd * len(deepstack_layers), n_patches_x * n_patches_y, batch_size]66    ggml_tensor * deepstack_features = nullptr;67    const int merge_factor = hparams.n_merge > 0 ? hparams.n_merge * hparams.n_merge : 4; // default 2x2=4 for qwen3vl68 69    // loop over layers70    for (int il = 0; il < n_layer; il++) {71        auto & layer = model.layers[il];72 73        ggml_tensor * cur = inpL; // inpL = residual, cur = hidden_states74 75        // layernorm176        cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);77        cb(cur, "ln1", il);78 79        // self-attention80        {81            cur = build_mm(layer.qkv_w, cur);82            cur = ggml_add(ctx0, cur, layer.qkv_b);83 84            ggml_tensor * Qcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos,85                    /* nb1    */ ggml_row_size(cur->type, d_head),86                    /* nb2    */ cur->nb[1],87                    /* offset */ 0);88 89            ggml_tensor * Kcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos,90                    /* nb1    */ ggml_row_size(cur->type, d_head),91                    /* nb2    */ cur->nb[1],92                    /* offset */ ggml_row_size(cur->type, n_embd));93 94            ggml_tensor * Vcur = ggml_view_3d(ctx0, cur, d_head, n_head, n_pos,95                    /* nb1    */ ggml_row_size(cur->type, d_head),96                    /* nb2    */ cur->nb[1],97                    /* offset */ ggml_row_size(cur->type, 2 * n_embd));98 99            cb(Qcur, "Qcur", il);100            cb(Kcur, "Kcur", il);101            cb(Vcur, "Vcur", il);102 103            // apply M-RoPE104            Qcur = ggml_rope_multi(105                ctx0, Qcur, positions, nullptr,106                d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);107            Kcur = ggml_rope_multi(108                ctx0, Kcur, positions, nullptr,109                d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION, 32768, 10000, 1, 0, 1, 32, 1);110 111            cb(Qcur, "Qcur_rope", il);112            cb(Kcur, "Kcur_rope", il);113 114            cur = build_attn(layer.o_w, layer.o_b,115                Qcur, Kcur, Vcur, nullptr, kq_scale, il);116            cb(cur, "attn_out", il);117        }118 119        // re-add the layer input, e.g., residual120        cur = ggml_add(ctx0, cur, inpL);121 122        inpL = cur; // inpL = residual, cur = hidden_states123 124        cb(cur, "ffn_inp", il);125 126        // layernorm2127        cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);128        cb(cur, "ffn_inp_normed", il);129 130        // ffn131        cur = build_ffn(cur,132            layer.ff_up_w, layer.ff_up_b,133            layer.ff_gate_w, layer.ff_gate_b,134            layer.ff_down_w, layer.ff_down_b,135            hparams.ffn_op, il);136 137        cb(cur, "ffn_out", il);138 139        // residual 2140        cur = ggml_add(ctx0, inpL, cur);141        cb(cur, "layer_out", il);142 143        if (layer.has_deepstack()) {144            ggml_tensor * feat = ggml_reshape_3d(ctx0, cur, n_embd * merge_factor, n_pos / merge_factor, batch_size);145            feat = build_norm(feat, layer.deepstack_norm_w, layer.deepstack_norm_b, norm_t, eps, il);146            feat = build_ffn(feat,147                layer.deepstack_fc1_w, layer.deepstack_fc1_b,148                nullptr, nullptr,149                layer.deepstack_fc2_w, layer.deepstack_fc2_b,150                ffn_op_type::FFN_GELU, il);151 152            if(!deepstack_features) {153                deepstack_features = feat;154            } else {155                // concat along the feature dimension156                deepstack_features = ggml_concat(ctx0, deepstack_features, feat, 0);157            }158        }159 160        inpL = cur;161    }162 163    // post-layernorm164    if (model.post_ln_w) {165        inpL = build_norm(inpL, model.post_ln_w, model.post_ln_b, norm_t, eps, n_layer);166    }167 168    // multimodal projection169    ggml_tensor * embeddings = inpL;170    embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);171 172    embeddings = build_ffn(embeddings,173        model.mm_0_w, model.mm_0_b,174        nullptr, nullptr,175        model.mm_1_w, model.mm_1_b,176        ffn_op_type::FFN_GELU, -1);177 178    if (deepstack_features) {179        embeddings = ggml_concat(ctx0, embeddings, deepstack_features, 0);180    } // concat along the feature dimension181 182    // build the graph183    ggml_build_forward_expand(gf, embeddings);184 185    return gf;186}187