Brunobkr/llama.cpp_AlgMor24_github

1#include "models.h"2 3ggml_tensor * clip_graph_minimax_m3::apply_rope(4        ggml_tensor * x, ggml_tensor * pos_h, ggml_tensor * pos_w) {5    const int64_t Hn  = x->ne[1];6    const int64_t P   = x->ne[2];7    const size_t  es  = ggml_element_size(x);8    const int     dh  = (int) x->ne[0];9    const int     axd = 2 * ((2 * (dh / 2) / 3) / 2);10 11    GGML_ASSERT(x->nb[0] == es);12    GGML_ASSERT(3 * axd <= dh);13 14    const float th  = hparams.rope_theta;15 16    // layout of x is [t, h, w, pad]17    // t is unrotated, h and w are rotated, pad is unrotated18    // note: everything from n_dims onward untouched, so w and pad are rotated in one call.19    auto sl = [&](int off, int n) {20        return ggml_cont(ctx0, ggml_view_3d(ctx0, x, n, Hn, P, x->nb[1], x->nb[2], (size_t) off * es));21    };22    ggml_tensor * t = sl(0,       axd);23    ggml_tensor * h = sl(axd,     axd);24    ggml_tensor * w = sl(2 * axd, dh - 2 * axd); // w + pad25 26    h = ggml_rope_ext(ctx0, h, pos_h, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);27    w = ggml_rope_ext(ctx0, w, pos_w, nullptr, axd, GGML_ROPE_TYPE_NEOX, 0, th, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);28    return ggml_concat(ctx0, ggml_concat(ctx0, t, h, 0), w, 0);29}30 31ggml_cgraph * clip_graph_minimax_m3::build() {32    GGML_ASSERT(model.patch_bias     == nullptr);33    GGML_ASSERT(model.class_embedding == nullptr);34    GGML_ASSERT(model.patch_embeddings_0 && model.patch_embeddings_1);35    GGML_ASSERT(model.mm_1_w && model.mm_2_w);36    GGML_ASSERT(model.mm_merger_fc1_w && model.mm_merger_fc2_w);37 38    const int batch_size = 1;39    const int n_pos      = n_patches;40    const int merge      = hparams.n_merge;41 42    // patch embedding43    ggml_tensor * inp_raw = build_inp_raw();44    ggml_tensor * inp = ggml_add(ctx0,45        ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1),46        ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));47 48    // spatial merge49    {50        inp = ggml_permute(ctx0, inp, 1, 2, 0, 3);51        inp = ggml_cont_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, n_patches_y, batch_size);52        inp = ggml_reshape_4d(ctx0, inp, n_embd * merge, n_patches_x / merge, merge, batch_size * (n_patches_y / merge));53        inp = ggml_permute(ctx0, inp, 0, 2, 1, 3);54        inp = ggml_cont_3d(ctx0, inp, n_embd, n_patches_x * n_patches_y, batch_size);55    }56 57    // t (time axis) is always 0 for now, so we leave it unrotated58    ggml_tensor * pos_h = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);59    ggml_set_name(pos_h, "minimax_pos_h"); ggml_set_input(pos_h);60    ggml_tensor * pos_w = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);61    ggml_set_name(pos_w, "minimax_pos_w"); ggml_set_input(pos_w);62 63    ggml_tensor * inpL = build_vit(64        inp, n_pos, NORM_TYPE_NORMAL, FFN_GELU_ERF, nullptr,65        [&](ggml_tensor * c, const clip_layer &) {66            return apply_rope(c, pos_h, pos_w);67        });68 69    // projector70    ggml_tensor * emb = inpL;71    emb = build_ffn(emb, model.mm_1_w, model.mm_1_b,72                    nullptr, nullptr,73                    model.mm_2_w, model.mm_2_b, FFN_GELU_ERF, -1);74 75    const int64_t proj = emb->ne[0];76    emb = ggml_reshape_2d(ctx0, emb, proj * merge * merge, n_pos / (merge * merge));77 78    emb = build_ffn(emb, model.mm_merger_fc1_w, model.mm_merger_fc1_b,79                    nullptr, nullptr,80                    model.mm_merger_fc2_w, model.mm_merger_fc2_b, FFN_GELU_ERF, -1);81 82    ggml_build_forward_expand(gf, emb);83    return gf;84}85