Brunobkr/llama.cpp_AlgMor24_github

1#include "models.h"2 3ggml_cgraph * clip_graph_step3vl::build() {4    GGML_ASSERT(model.class_embedding == nullptr);5    GGML_ASSERT(model.patch_embeddings_0 != nullptr);6    GGML_ASSERT(model.position_embeddings != nullptr);7 8    norm_type norm_t = NORM_TYPE_NORMAL;9 10    ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);11    ggml_set_name(pos_h, "pos_h");12    ggml_set_input(pos_h);13 14    ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_patches);15    ggml_set_name(pos_w, "pos_w");16    ggml_set_input(pos_w);17 18    ggml_tensor * inp = build_inp();19    ggml_tensor * learned_pos_embd = resize_position_embeddings();20 21    auto add_pos = [&](ggml_tensor * cur, const clip_layer &) {22        return build_rope_2d(ctx0, cur, pos_w, pos_h, hparams.rope_theta, false);23    };24 25    auto add_spatial_bias = [&](ggml_tensor * cur, ggml_tensor * bias) {26        if (bias == nullptr) {27            return cur;28        }29 30        const int64_t width    = cur->ne[0];31        const int64_t height   = cur->ne[1];32        const int64_t channels = cur->ne[2];33 34        cur = ggml_reshape_2d(ctx0, cur, width * height, channels);35        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));36        cur = ggml_add(ctx0, cur, bias);37        cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));38        cur = ggml_reshape_3d(ctx0, cur, width, height, channels);39 40        return cur;41    };42 43    ggml_tensor * cur = build_vit(44        inp,45        n_patches,46        norm_t,47        hparams.ffn_op,48        learned_pos_embd,49        add_pos);50    cb(cur, "vit_out", -1);51 52    // [n_embd, n_patches] -> [w, h, n_embd] for spatial downsampling convolutions.53    cur = ggml_permute(ctx0, cur, 1, 0, 2, 3);54    cur = ggml_cont_3d(ctx0, cur, n_patches_x, n_patches_y, n_embd);55 56    // First downsampler: Conv2d(1536 -> 3072, k=3, s=2, p=1)57    cur = ggml_conv_2d(ctx0, model.mm_0_w, cur, 2, 2, 1, 1, 1, 1);58    cur = add_spatial_bias(cur, model.mm_0_b);59    cb(cur, "downsample_0", -1);60 61    // Second downsampler: Conv2d(3072 -> 6144, k=3, s=2, p=1)62    cur = ggml_conv_2d(ctx0, model.mm_1_w, cur, 2, 2, 1, 1, 1, 1);63    cur = add_spatial_bias(cur, model.mm_1_b);64    cb(cur, "downsample_1", -1);65 66    // [w, h, c] -> [c, w*h]67    {68        const int64_t w = cur->ne[0];69        const int64_t h = cur->ne[1];70        cur = ggml_reshape_3d(ctx0, cur, w * h, cur->ne[2], cur->ne[3]);71        cur = ggml_cont(ctx0, ggml_permute(ctx0, cur, 1, 0, 2, 3));72    }73    cb(cur, "downsample_flatten", -1);74 75    // Final projector: Linear(6144 -> projection_dim)76    cur = ggml_mul_mat(ctx0, model.mm_model_proj, cur);77    cb(cur, "projector_out", -1);78 79    ggml_build_forward_expand(gf, cur);80    return gf;81}82