Brunobkr/llama.cpp_AlgMor24_github

1#include "models.h"2 3void llama_model_mimo2::load_arch_hparams(llama_model_loader & ml) {4    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);5 6    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;7 8    ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp);9    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW,   hparams.n_swa);10    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA,         hparams.rope_freq_base_train_swa, false);11 12    ml.get_key_or_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl, hparams.n_layer());13 14    float value_scale = 0.0f;15    if (ml.get_key(LLM_KV_ATTENTION_VALUE_SCALE, value_scale, false) && value_scale != 1.0f) {16        hparams.f_attn_value_scale = value_scale;17    }18 19    ml.get_key(LLM_KV_NEXTN_PREDICT_LAYERS, hparams.n_layer_nextn, false);20    GGML_ASSERT(hparams.n_layer_nextn < hparams.n_layer_all && "n_layer_nextn must be < n_layer_impl");21 22    switch (hparams.n_layer()) {23        case 48: type = LLM_TYPE_310B_A15B; break;24        default: type = LLM_TYPE_UNKNOWN;25    }26}27 28void llama_model_mimo2::load_arch_tensors(llama_model_loader & ml) {29    LLAMA_LOAD_LOCALS;30 31    const std::string mtp_probe = "blk." + std::to_string(n_layer) + ".nextn.eh_proj.weight";32    const bool trunk_only = (hparams.n_layer_nextn > 0) && (ml.get_weight(mtp_probe.c_str()) == nullptr);33    int mtp_flags         = trunk_only ? TENSOR_NOT_REQUIRED : 0;34 35    if (!ml.load_mtp) {36        mtp_flags |= TENSOR_SKIP;37    }38 39    tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0);40 41    // output42    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);43    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, 0);44 45    for (int i = 0; i < n_layer_all; ++i) {46        auto & layer = layers[i];47        uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(i);48        uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(i);49        uint32_t n_head = hparams.n_head(i);50 51        const bool is_nextn = i >= n_layer;52        const int  flags    = is_nextn ? mtp_flags : 0;53 54        create_tensor_qkv(layer, i, n_embd, n_embd_head_k * n_head, n_embd_k_gqa, n_embd_v_gqa, flags);55        layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), { n_embd_head_v * n_head, n_embd }, flags);56 57        layer.attn_norm  = create_tensor(tn(LLM_TENSOR_ATTN_NORM,  "weight", i), {n_embd}, flags);58        layer.attn_sinks = create_tensor(tn(LLM_TENSOR_ATTN_SINKS, "weight", i), {n_head}, TENSOR_NOT_REQUIRED | flags);59 60        layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, flags);61 62        // non-MoE branch63        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd,   n_ff}, TENSOR_NOT_REQUIRED | flags);64        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  n_ff, n_embd}, TENSOR_NOT_REQUIRED | flags);65        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd,   n_ff}, TENSOR_NOT_REQUIRED | flags);66 67        // MoE branch68        int64_t n_ff_exp = hparams.n_ff_exp;69        layer.ffn_gate_inp  = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP,  "weight", i), {n_embd, n_expert}, TENSOR_NOT_REQUIRED | flags);70        layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff_exp,   n_expert}, TENSOR_NOT_REQUIRED | flags);71        layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp,   n_embd, n_expert}, TENSOR_NOT_REQUIRED | flags);72        layer.ffn_up_exps   = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS,   "weight", i), {n_embd, n_ff_exp,   n_expert}, TENSOR_NOT_REQUIRED | flags);73        layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED | flags);74 75        if (is_nextn) {76            layer.nextn.eh_proj          = create_tensor(tn(LLM_TENSOR_NEXTN_EH_PROJ,          "weight", i), {2 * n_embd, n_embd}, flags);77            layer.nextn.enorm            = create_tensor(tn(LLM_TENSOR_NEXTN_ENORM,            "weight", i), {n_embd}, flags);78            layer.nextn.hnorm            = create_tensor(tn(LLM_TENSOR_NEXTN_HNORM,            "weight", i), {n_embd}, flags);79            layer.nextn.embed_tokens     = create_tensor(tn(LLM_TENSOR_NEXTN_EMBED_TOKENS,     "weight", i), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED | flags);80            layer.nextn.shared_head_head = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_HEAD, "weight", i), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED | flags);81            layer.nextn.shared_head_norm = create_tensor(tn(LLM_TENSOR_NEXTN_SHARED_HEAD_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED | flags);82            layer.layer_out_norm         = create_tensor(tn(LLM_TENSOR_LAYER_OUT_NORM,         "weight", i), {n_embd}, TENSOR_NOT_REQUIRED | flags);83        }84    }85}86 87std::unique_ptr<llm_graph_context> llama_model_mimo2::build_arch_graph(const llm_graph_params & params) const {88    if (params.gtype == LLM_GRAPH_TYPE_DECODER_MTP) {89        return std::make_unique<graph_mtp>(*this, params);90    }91    return std::make_unique<graph>(*this, params);92}93 94llama_model_mimo2::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {95    ggml_tensor * cur;96    ggml_tensor * inpL;97 98    inpL = build_inp_embd(model.tok_embd);99 100    ggml_tensor * inp_pos = build_inp_pos();101    auto * inp_attn = build_attn_inp_kv_iswa();102    ggml_tensor * inp_out_ids = build_inp_out_ids();103 104    const float v_scale = hparams.f_attn_value_scale;105    const bool emit_h_nextn = cparams.embeddings_nextn;106    const bool crop_last_layer = inp_out_ids && (!emit_h_nextn || cparams.embeddings_nextn_masked);107 108    for (int il = 0; il < n_layer; ++il) {109        ggml_tensor * inpSA = inpL;110 111        uint32_t n_head_l    = hparams.n_head(il);112        uint32_t n_head_kv_l = hparams.n_head_kv(il);113        const float freq_base_l  = model.get_rope_freq_base(cparams, il);114        const float freq_scale_l = model.get_rope_freq_scale(cparams, il);115 116        cur = inpL;117 118        // self_attention119        {120            cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);121            cb(cur, "attn_norm", il);122 123            ggml_tensor * Qcur;124            ggml_tensor * Kcur;125            ggml_tensor * Vcur;126 127            if (model.layers[il].wqkv) {128                // Fused qkv_proj - Q/K share head_dim_k, V uses head_dim_v129                ggml_tensor * qkv = build_lora_mm(model.layers[il].wqkv, cur);130                cb(qkv, "wqkv", il);131 132                const size_t row_k    = ggml_row_size(qkv->type, n_embd_head_k);133                const size_t row_v    = ggml_row_size(qkv->type, n_embd_head_v);134                const size_t row_full = qkv->nb[1];135                const size_t k_off    = row_k * n_head_l;136                const size_t v_off    = k_off + row_k * n_head_kv_l;137 138                Qcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_l,    n_tokens, row_k, row_full, 0);139                Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_kv_l, n_tokens, row_k, row_full, k_off);140                Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_kv_l, n_tokens, row_v, row_full, v_off);141            } else {142                // Split path143                Qcur = build_lora_mm(model.layers[il].wq, cur);144                cb(Qcur, "Qcur", il);145 146                Kcur = build_lora_mm(model.layers[il].wk, cur);147                cb(Kcur, "Kcur", il);148 149                Vcur = build_lora_mm(model.layers[il].wv, cur);150                cb(Vcur, "Vcur", il);151 152                Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head_k, n_head_l,    n_tokens);153                Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head_k, n_head_kv_l, n_tokens);154                Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head_v, n_head_kv_l, n_tokens);155            }156 157            Qcur = ggml_rope_ext(158                ctx0, Qcur, inp_pos, nullptr,159                n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,160                ext_factor, attn_factor, beta_fast, beta_slow161                );162 163            Kcur = ggml_rope_ext(164                ctx0, Kcur, inp_pos, nullptr,165                n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,166                ext_factor, attn_factor, beta_fast, beta_slow167                );168 169            cb(Qcur, "Qcur", il);170            cb(Kcur, "Kcur", il);171            cb(Vcur, "Vcur", il);172 173            ggml_tensor * sinks = model.layers[il].attn_sinks;174 175            cur = build_attn(inp_attn,176                    model.layers[il].wo, NULL, model.layers[il].wo_s,177                    Qcur, Kcur, Vcur, nullptr, sinks, nullptr, 1.0f/sqrtf(float(n_embd_head_k)), il);178            cb(cur, "attn_out", il);179 180            if (v_scale) {181                cur = ggml_scale(ctx0, cur, v_scale);182                cb(cur, "attn_out_scaled", il);183            }184        }185 186        if (il == n_layer - 1 && crop_last_layer) {187            cur   = ggml_get_rows(ctx0,   cur, inp_out_ids);188            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);189        }190 191        ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);192        cb(ffn_inp, "ffn_inp", il);193 194        cur = build_norm(ffn_inp,195                model.layers[il].ffn_norm, NULL,196                LLM_NORM_RMS, il);197        cb(cur, "ffn_norm", il);198 199        // feed-forward network200        if (model.layers[il].ffn_gate_inp == nullptr) {201            // dense branch202            cur = build_ffn(cur,203                    model.layers[il].ffn_up,   model.layers[il].ffn_up_b,   NULL,204                    model.layers[il].ffn_gate, model.layers[il].ffn_gate_b, NULL,205                    model.layers[il].ffn_down, model.layers[il].ffn_down_b, NULL,206                    NULL,207                    LLM_FFN_SILU, LLM_FFN_PAR, il);208            cb(cur, "ffn_out", il);209        } else {210            // MoE branch211            cur = build_moe_ffn(cur,212                    model.layers[il].ffn_gate_inp,213                    model.layers[il].ffn_up_exps,214                    model.layers[il].ffn_gate_exps,215                    model.layers[il].ffn_down_exps,216                    model.layers[il].ffn_exp_probs_b,217                    n_expert, n_expert_used,218                    LLM_FFN_SILU, true,219                    hparams.expert_weights_scale,220                    LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID,221                    il);222            cb(cur, "ffn_moe_out", il);223        }224 225        cur = ggml_add(ctx0, cur, ffn_inp);226 227        cur = build_cvec(cur, il);228        cb(cur, "l_out", il);229 230        // input for next layer231        inpL = cur;232    }233 234    cur = inpL;235 236    if (emit_h_nextn) {237        cb(cur, "h_nextn", -1);238        res->t_h_nextn = cur;239 240        if (!cparams.embeddings_nextn_masked && inp_out_ids) {241            cur = ggml_get_rows(ctx0, cur, inp_out_ids);242        }243    }244 245    cur = build_norm(cur,246            model.output_norm, NULL,247            LLM_NORM_RMS, -1);248 249    cb(cur, "result_norm", -1);250    res->t_embd = cur;251 252    // lm_head253    cur = build_lora_mm(model.output, cur, model.output_s);254 255    cb(cur, "result_output", -1);256    res->t_logits = cur;257 258    ggml_build_forward_expand(gf, cur);259}260 261// Mirrors MiMo's appended NextN block: normalize and fuse token and hidden inputs, run the decoder block,262// expose its pre-head-norm state to the next draft step, then apply the shared output norm and LM head.263// Converted checkpoints may store that shared norm as layer_out_norm, so it remains in the fallback chain.264llama_model_mimo2::graph_mtp::graph_mtp(const llama_model & model, const llm_graph_params & params)265    : llm_graph_context(params) {266    GGML_ASSERT(hparams.n_layer_nextn > 0 && "MIMO2 MTP requires n_layer_nextn > 0");267 268    const int il = hparams.n_layer() + cparams.nextn_layer_offset;269    GGML_ASSERT(cparams.nextn_layer_offset >= 0 &&270                cparams.nextn_layer_offset < (int) hparams.n_layer_nextn &&271                "nextn_layer_offset out of range [0, n_layer_nextn)");272 273    const auto & layer = model.layers[il];274    GGML_ASSERT(layer.nextn.eh_proj && "MIMO2 MTP block missing nextn.eh_proj");275    GGML_ASSERT(layer.nextn.enorm   && "MIMO2 MTP block missing nextn.enorm");276    GGML_ASSERT(layer.nextn.hnorm   && "MIMO2 MTP block missing nextn.hnorm");277    GGML_ASSERT(layer.wqkv          && "MIMO2 MTP requires fused attn_qkv");278 279    const uint32_t n_head_l    = hparams.n_head(il);280    const uint32_t n_head_kv_l = hparams.n_head_kv(il);281 282    const float freq_base_l  = model.get_rope_freq_base(cparams, il);283    const float freq_scale_l = model.get_rope_freq_scale(cparams, il);284    const float v_scale      = hparams.f_attn_value_scale;285 286    auto inp = std::make_unique<llm_graph_input_embd>(hparams.n_embd);287 288    inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);289    ggml_set_input(inp->tokens);290 291    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);292    ggml_set_input(inp->embd);293    ggml_set_name(inp->embd, "mtp_h_input");294 295    ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;296    ggml_tensor * h_input    = inp->embd;297    ggml_tensor * tok_embd   = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);298    cb(tok_embd, "mtp_tok_embd", il);299 300    res->add_input(std::move(inp));301 302    ggml_tensor * inp_pos     = build_inp_pos();303    ggml_tensor * inp_out_ids = build_inp_out_ids();304    auto        * inp_attn    = build_attn_inp_kv_iswa();305 306    ggml_tensor * h_norm = build_norm(h_input, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);307    cb(h_norm, "mtp_hnorm", il);308 309    ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);310    cb(e_norm, "mtp_enorm", il);311 312    ggml_tensor * concat = ggml_concat(ctx0, e_norm, h_norm, /*dim=*/ 0);313    cb(concat, "mtp_concat", il);314 315    ggml_tensor * cur = build_lora_mm(layer.nextn.eh_proj, concat, layer.nextn.eh_proj_s);316    cb(cur, "mtp_eh_proj", il);317 318    ggml_tensor * inpSA = cur;319 320    cur = build_norm(cur, layer.attn_norm, nullptr, LLM_NORM_RMS, il);321    cb(cur, "mtp_attn_norm", il);322 323    ggml_tensor * qkv = build_lora_mm(layer.wqkv, cur, layer.wqkv_s);324    cb(qkv, "mtp_wqkv", il);325 326    const size_t row_k    = ggml_row_size(qkv->type, n_embd_head_k);327    const size_t row_v    = ggml_row_size(qkv->type, n_embd_head_v);328    const size_t row_full = qkv->nb[1];329    const size_t k_off    = row_k * n_head_l;330    const size_t v_off    = k_off + row_k * n_head_kv_l;331 332    ggml_tensor * Qcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_l,    n_tokens, row_k, row_full, 0);333    ggml_tensor * Kcur = ggml_view_3d(ctx0, qkv, n_embd_head_k, n_head_kv_l, n_tokens, row_k, row_full, k_off);334    ggml_tensor * Vcur = ggml_view_3d(ctx0, qkv, n_embd_head_v, n_head_kv_l, n_tokens, row_v, row_full, v_off);335 336    Qcur = ggml_rope_ext(337        ctx0, Qcur, inp_pos, nullptr,338        n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,339        ext_factor, attn_factor, beta_fast, beta_slow);340 341    Kcur = ggml_rope_ext(342        ctx0, Kcur, inp_pos, nullptr,343        n_rot, rope_type, n_ctx_orig, freq_base_l, freq_scale_l,344        ext_factor, attn_factor, beta_fast, beta_slow);345 346    cb(Qcur, "mtp_Qcur", il);347    cb(Kcur, "mtp_Kcur", il);348    cb(Vcur, "mtp_Vcur", il);349 350    cur = build_attn(inp_attn,351            layer.wo, nullptr, layer.wo_s,352            Qcur, Kcur, Vcur, nullptr, layer.attn_sinks, nullptr,353            1.0f / sqrtf(float(n_embd_head_k)), il);354    cb(cur, "mtp_attn_out", il);355 356    if (v_scale) {357        cur = ggml_scale(ctx0, cur, v_scale);358        cb(cur, "mtp_attn_out_scaled", il);359    }360 361    ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA);362    cb(ffn_inp, "mtp_ffn_inp", il);363 364    cur = build_norm(ffn_inp, layer.ffn_norm, nullptr, LLM_NORM_RMS, il);365    cb(cur, "mtp_ffn_norm", il);366 367    GGML_ASSERT(layer.ffn_gate && layer.ffn_down && layer.ffn_up && "MIMO2 MTP requires dense FFN tensors");368    cur = build_ffn(cur,369            layer.ffn_up,   layer.ffn_up_b,   nullptr,370            layer.ffn_gate, layer.ffn_gate_b, nullptr,371            layer.ffn_down, layer.ffn_down_b, nullptr,372            nullptr,373            LLM_FFN_SILU, LLM_FFN_PAR, il);374    cb(cur, "mtp_ffn_out", il);375 376    cur = ggml_add(ctx0, cur, ffn_inp);377    cb(cur, "mtp_post_ffn", il);378 379    cur = ggml_get_rows(ctx0, cur, inp_out_ids);380 381    cb(cur, "h_nextn", -1);382    res->t_h_nextn = cur;383 384    ggml_tensor * head_norm_w = layer.nextn.shared_head_norm385            ? layer.nextn.shared_head_norm386            : (layer.layer_out_norm ? layer.layer_out_norm : model.output_norm);387    GGML_ASSERT(head_norm_w && "MIMO2 MTP missing head norm fallback");388    cur = build_norm(cur, head_norm_w, nullptr, LLM_NORM_RMS, -1);389    cb(cur, "mtp_shared_head_norm", -1);390 391    ggml_tensor * head_w = layer.nextn.shared_head_head ? layer.nextn.shared_head_head : model.output;392    ggml_tensor * head_s = layer.nextn.shared_head_head ? layer.nextn.shared_head_head_s : model.output_s;393    GGML_ASSERT(head_w && "MIMO2 MTP missing LM head fallback");394    cur = build_lora_mm(head_w, cur, head_s);395    cb(cur, "result_output", -1);396 397    res->t_logits = cur;398    ggml_build_forward_expand(gf, cur);399}400