echodict/llama.cpp

1#include "models.h"2 3#include "llama-memory-recurrent.h"4 5llm_build_qwen35::llm_build_qwen35(const llama_model & model, const llm_graph_params & params) :6    llm_build_delta_net_base(params), model(model) {7    const int64_t n_embd_head = hparams.n_embd_head_v();8 9    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());10 11    int sections[4];12    std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);13 14    ggml_tensor * cur;15    ggml_tensor * inpL;16 17    inpL = build_inp_embd(model.tok_embd);18 19    cb(inpL, "model.input_embed", -1);20 21    auto * inp = build_inp_mem_hybrid();22 23    ggml_tensor * inp_pos     = build_inp_pos();24    ggml_tensor * inp_out_ids = build_inp_out_ids();25 26    for (int il = 0; il < n_layer; ++il) {27        ggml_tensor * inpSA = inpL;28 29        cur = build_norm(inpL, model.layers[il].attn_norm, nullptr, LLM_NORM_RMS, il);30        cb(cur, "attn_norm", il);31 32        ggml_build_forward_expand(gf, cur);33 34        // Determine layer type and build appropriate attention mechanism35        if (hparams.is_recurrent(il)) {36            // Linear attention layer (gated delta net)37            cur = build_layer_attn_linear(inp->get_recr(), cur, il);38        } else {39            // Full attention layer40            cur = build_layer_attn(inp->get_attn(), cur, inp_pos, sections, il);41        }42 43        if (il == n_layer - 1 && inp_out_ids) {44            cur   = ggml_get_rows(ctx0, cur, inp_out_ids);45            inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids);46        }47 48        // Residual connection49        cur = ggml_add(ctx0, cur, inpSA);50        cb(cur, "attn_residual", il);51 52        // Save the tensor before post-attention norm for residual connection53        ggml_tensor * ffn_residual = cur;54 55        // Post-attention norm56        ggml_tensor * attn_post_norm = build_norm(cur, model.layers[il].attn_post_norm, nullptr, LLM_NORM_RMS, il);57        cb(attn_post_norm, "attn_post_norm", il);58 59        // Dense FFN layer - without residual connection60        cur = build_layer_ffn(attn_post_norm, il);61        cb(cur, "ffn_out", il);62 63        // Residual connection for FFN - add to the tensor from before post_attention_layernorm64        cur = ggml_add(ctx0, cur, ffn_residual);65        cb(cur, "post_ffn", il);66 67        cur = build_cvec(cur, il);68        cb(cur, "l_out", il);69 70        // Input for next layer71        inpL = cur;72    }73    cur = inpL;74 75    // Final norm76    cur = build_norm(cur, model.output_norm, nullptr, LLM_NORM_RMS, -1);77 78    cb(cur, "result_norm", -1);79    res->t_embd = cur;80 81    // LM head82    cur = build_lora_mm(model.output, cur);83 84    cb(cur, "result_output", -1);85    res->t_logits = cur;86 87    ggml_build_forward_expand(gf, cur);88}89 90std::pair<ggml_tensor *, ggml_tensor *> llm_build_qwen35::build_qkvz(91                ggml_tensor * input,92                        int   il) {93    const int64_t n_seqs       = ubatch.n_seqs;94    const int64_t n_seq_tokens = ubatch.n_seq_tokens;95 96    ggml_tensor * qkv_mixed = build_lora_mm(model.layers[il].wqkv, input, model.layers[il].wqkv_s);97    qkv_mixed = ggml_reshape_3d(ctx0, qkv_mixed, qkv_mixed->ne[0], n_seq_tokens, n_seqs);98    cb(qkv_mixed, "linear_attn_qkv_mixed", il);99 100    ggml_tensor * z = build_lora_mm(model.layers[il].wqkv_gate, input, model.layers[il].wqkv_gate_s);101    cb(z, "z", il);102 103    return { qkv_mixed, z };104}105 106ggml_tensor * llm_build_qwen35::build_norm_gated(107        ggml_tensor * input,108        ggml_tensor * weights,109        ggml_tensor * gate,110        int           layer) {111    ggml_tensor * normalized = build_norm(input, weights, nullptr, LLM_NORM_RMS, layer);112    ggml_tensor * gated_silu = ggml_silu(ctx0, gate);113 114    return ggml_mul(ctx0, normalized, gated_silu);115}116 117ggml_tensor * llm_build_qwen35::build_layer_attn(118        llm_graph_input_attn_kv * inp,119        ggml_tensor *             cur,120        ggml_tensor *             inp_pos,121        int *                     sections,122        int                       il) {123    const int64_t n_embd_head = hparams.n_embd_head_v();124    GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());125 126    // Order: joint QG projection, QG split, Q norm, KV projection, K norm, RoPE, attention127 128    // Qwen3Next uses a single Q projection that outputs query + gate129    ggml_tensor * Qcur_full = build_lora_mm(model.layers[il].wq, cur, model.layers[il].wq_s); // [ (n_embd_head * 2) * n_head, n_tokens ]130    cb(Qcur_full, "Qcur_full", il);131 132    ggml_tensor * Qcur = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens,133        ggml_element_size(Qcur_full) * n_embd_head * 2,134        ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head, 0);135    cb(Qcur, "Qcur_reshaped", il);136 137    // Apply Q normalization138    Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, nullptr, LLM_NORM_RMS, il);139    cb(Qcur, "Qcur_normed", il);140 141    ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur, model.layers[il].wk_s);142    cb(Kcur, "Kcur", il);143 144    ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur, model.layers[il].wv_s);145    cb(Vcur, "Vcur", il);146 147    // Apply K normalization148    Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv, n_tokens);149    Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, nullptr, LLM_NORM_RMS, il);150    cb(Kcur, "Kcur_normed", il);151 152    ggml_tensor * gate = ggml_view_3d(ctx0, Qcur_full, n_embd_head, n_head, n_tokens,153        ggml_element_size(Qcur_full) * n_embd_head * 2,154        ggml_element_size(Qcur_full) * n_embd_head * 2 * n_head,155        ggml_element_size(Qcur_full) * n_embd_head);156    gate = ggml_cont_2d(ctx0, gate, n_embd_head * n_head, n_tokens);157    cb(gate, "gate_reshaped", il);158 159    Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv, n_tokens);160 161    // Apply MRoPE162    Qcur = ggml_rope_multi(163            ctx0, Qcur, inp_pos, nullptr,164            n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,165            ext_factor, attn_factor, beta_fast, beta_slow166            );167 168    Kcur = ggml_rope_multi(169            ctx0, Kcur, inp_pos, nullptr,170            n_rot, sections, rope_type, n_ctx_orig, freq_base, freq_scale,171            ext_factor, attn_factor, beta_fast, beta_slow172            );173 174    cb(Qcur, "Qcur", il);175    cb(Kcur, "Kcur", il);176    cb(Vcur, "Vcur", il);177 178    // Attention computation179    const float kq_scale = hparams.f_attention_scale == 0.0f ? 1.0f / sqrtf(float(n_embd_head)) : hparams.f_attention_scale;180 181    cur = build_attn(inp,182                nullptr, nullptr, nullptr,183                Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il);184    cb(cur, "attn_pregate", il);185 186    ggml_tensor * gate_sigmoid = ggml_sigmoid(ctx0, gate);187    cb(gate_sigmoid, "gate_sigmoid", il);188 189    cur = ggml_mul(ctx0, cur, gate_sigmoid);190    cb(cur, "attn_gated", il);191 192    cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s);193    cb(cur, "attn_output", il);194 195    return cur;196}197 198ggml_tensor * llm_build_qwen35::build_layer_attn_linear(199        llm_graph_input_rs * inp,200        ggml_tensor *        cur,201        int                  il) {202    const auto * mctx_cur = inp->mctx;203 204    const int64_t d_inner      = hparams.ssm_d_inner;205    const int64_t n_seqs       = ubatch.n_seqs;206    const int64_t head_k_dim   = hparams.ssm_d_state;207    const int64_t num_k_heads  = hparams.ssm_n_group;208    const int64_t num_v_heads  = hparams.ssm_dt_rank;209    const int64_t head_v_dim   = d_inner / num_v_heads;210    const int64_t n_seq_tokens = ubatch.n_seq_tokens;211 212    const auto kv_head = mctx_cur->get_head();213 214    GGML_ASSERT(n_seqs != 0);215    GGML_ASSERT(ubatch.equal_seqs());216    GGML_ASSERT(ubatch.n_tokens == n_seq_tokens * n_seqs);217 218    // Input projections219    auto qkvz = build_qkvz(cur, il);220    ggml_tensor * qkv_mixed = qkvz.first;221    ggml_tensor * z         = qkvz.second;222 223    ggml_tensor * beta = build_lora_mm(model.layers[il].ssm_beta, cur, model.layers[il].ssm_beta_s);224    beta = ggml_reshape_4d(ctx0, beta, 1, num_v_heads, n_seq_tokens, n_seqs);225    cb(beta, "beta", il);226 227    beta = ggml_sigmoid(ctx0, beta);228    cb(beta, "beta_sigmoid", il);229 230    ggml_tensor * alpha = build_lora_mm(model.layers[il].ssm_alpha, cur, model.layers[il].ssm_alpha_s);231    alpha = ggml_reshape_3d(ctx0, alpha, num_v_heads, n_seq_tokens, n_seqs);232    cb(alpha, "alpha", il);233 234    ggml_tensor * alpha_biased   = ggml_add(ctx0, alpha, model.layers[il].ssm_dt);235    ggml_tensor * alpha_softplus = ggml_softplus(ctx0, alpha_biased);236    cb(alpha_softplus, "a_softplus", il);237 238    ggml_tensor * gate = ggml_mul(ctx0, alpha_softplus, model.layers[il].ssm_a);  // -A_log.exp() * softplus239    cb(gate, "gate", il);240 241    gate = ggml_reshape_4d(ctx0, gate, 1, num_v_heads, n_seq_tokens, n_seqs);242 243    // Get convolution states from cache244    ggml_tensor * conv_states_all = mctx_cur->get_r_l(il);245    ggml_tensor * ssm_states_all  = mctx_cur->get_s_l(il);246 247    // Build the convolution states tensor248    ggml_tensor * conv_states = build_rs(inp, conv_states_all, hparams.n_embd_r(), n_seqs);249    cb(conv_states, "conv_states", il);250 251    // Calculate convolution kernel size252    ggml_tensor * conv_kernel      = model.layers[il].ssm_conv1d;253    const int64_t conv_kernel_size = conv_kernel->ne[0];254    const int64_t conv_channels    = d_inner + 2 * hparams.ssm_n_group * hparams.ssm_d_state;255 256    conv_states = ggml_reshape_3d(ctx0, conv_states, conv_kernel_size - 1, conv_channels, n_seqs);257    cb(conv_states, "conv_states_reshaped", il);258 259    qkv_mixed = ggml_transpose(ctx0, qkv_mixed);260    cb(qkv_mixed, "qkv_mixed_transposed", il);261 262    ggml_tensor * conv_input = ggml_concat(ctx0, conv_states, qkv_mixed, 0);263    cb(conv_input, "conv_input", il);264 265    // Update convolution state cache266    // Extract the last (conv_kernel_size - 1) states from conv_input267    ggml_tensor * last_conv_states =268        ggml_view_3d(ctx0, conv_input, conv_kernel_size - 1, conv_channels, n_seqs, conv_input->nb[1],269                     conv_input->nb[2], (conv_input->ne[0] - conv_states->ne[0]) * ggml_element_size(conv_input));270    cb(last_conv_states, "last_conv_states", il);271 272    ggml_tensor * state_update_target =273        ggml_view_2d(ctx0, conv_states_all, (conv_kernel_size - 1) * conv_channels, n_seqs, conv_states_all->nb[1],274                     kv_head * (conv_kernel_size - 1) * conv_channels * ggml_element_size(conv_states_all));275    cb(state_update_target, "state_update_target", il);276 277    ggml_build_forward_expand(gf, ggml_cpy(ctx0, last_conv_states, state_update_target));278 279    ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs);280    state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs);281    cb(state, "state_predelta", il);282 283    ggml_tensor * conv_output_proper = ggml_ssm_conv(ctx0, conv_input, conv_kernel);284    cb(conv_output_proper, "conv_output_raw", il);285 286    ggml_tensor * conv_output_silu = ggml_silu(ctx0, conv_output_proper);287    cb(conv_output_silu, "conv_output_silu", il);288 289    ggml_tensor * conv_qkv_mix = conv_output_silu;290 291    // Calculate the total conv dimension292    int64_t qkv_dim = head_k_dim * num_k_heads * 2 + head_v_dim * num_v_heads;293    int64_t nb1_qkv = ggml_row_size(conv_qkv_mix->type, qkv_dim);294 295    // Extract the convolved Q, K, V from conv_output296    ggml_tensor * q_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,297            ggml_row_size(conv_qkv_mix->type, head_k_dim),298            nb1_qkv,299            nb1_qkv * n_seq_tokens,300            0);301 302    ggml_tensor * k_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_k_dim, num_k_heads, n_seq_tokens, n_seqs,303            ggml_row_size(conv_qkv_mix->type, head_k_dim),304            nb1_qkv,305            nb1_qkv * n_seq_tokens,306            head_k_dim * num_k_heads * ggml_element_size(conv_qkv_mix));307 308    ggml_tensor * v_conv = ggml_view_4d(ctx0, conv_qkv_mix, head_v_dim, num_v_heads, n_seq_tokens, n_seqs,309            ggml_row_size(conv_qkv_mix->type, head_v_dim),310            nb1_qkv,311            nb1_qkv * n_seq_tokens,312            ggml_row_size(conv_qkv_mix->type, 2 * head_k_dim * num_k_heads));313 314    cb(q_conv, "q_conv", il);315    cb(k_conv, "k_conv", il);316    cb(v_conv, "v_conv", il);317 318    const float eps_norm = hparams.f_norm_rms_eps;319 320    q_conv = ggml_l2_norm(ctx0, q_conv, eps_norm);321    k_conv = ggml_l2_norm(ctx0, k_conv, eps_norm);322 323    //q_conv = ggml_cont_4d(ctx0, q_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs);324    //k_conv = ggml_cont_4d(ctx0, k_conv, head_k_dim, num_k_heads, n_seq_tokens, n_seqs);325    //v_conv = ggml_cont_4d(ctx0, v_conv, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);326 327    // if head keys and value keys are different, repeat to force tensors into matching shapes328    // note: need explicit repeat only if we are not using the fused GDN329    if (num_k_heads != num_v_heads && (!cparams.fused_gdn_ar || !cparams.fused_gdn_ch)) {330        GGML_ASSERT(num_v_heads % num_k_heads == 0);331        q_conv = ggml_repeat_4d(ctx0, q_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);332        k_conv = ggml_repeat_4d(ctx0, k_conv, head_k_dim, num_v_heads, n_seq_tokens, n_seqs);333    }334 335    cb(q_conv, "q_conv_predelta", il);336    cb(k_conv, "k_conv_predelta", il);337    cb(v_conv, "v_conv_predelta", il);338 339    auto attn_out = build_delta_net(q_conv, k_conv, v_conv, gate, beta, state, il);340 341    ggml_tensor * output    = attn_out.first;342    ggml_tensor * new_state = attn_out.second;343    cb(output, "attn_output", il);344    cb(new_state, "new_state", il);345 346    // Update the recurrent states347    ggml_build_forward_expand(gf,348            ggml_cpy(ctx0, new_state,349                ggml_view_2d(ctx0, ssm_states_all, hparams.n_embd_s(), n_seqs, ssm_states_all->nb[1],350                    kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all))));351 352    // z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim]353    ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);354 355    // Apply gated normalization: self.norm(core_attn_out, z)356    ggml_tensor * attn_out_norm = build_norm_gated(output, model.layers[il].ssm_norm, z_2d, il);357 358    // Final reshape: [head_dim, n_heads, n_tokens, n_seqs] -> [n_tokens, n_seqs, n_heads * head_dim]359    ggml_tensor * final_output = ggml_reshape_3d(ctx0, attn_out_norm, head_v_dim * num_v_heads, n_seq_tokens, n_seqs);360    cb(final_output, "final_output", il);361 362    // Output projection363    cur = build_lora_mm(model.layers[il].ssm_out, final_output, model.layers[il].ssm_out_s);364    cb(cur, "linear_attn_out", il);365 366    // Reshape back to original dimensions367    cur = ggml_reshape_2d(ctx0, cur, n_embd, n_seq_tokens * n_seqs);368 369    return cur;370}371 372ggml_tensor * llm_build_qwen35::build_layer_ffn(ggml_tensor * cur, const int il) {373    // Qwen3.5 does not use MoE FFN374    GGML_ASSERT(model.layers[il].ffn_gate_inp == nullptr);375 376    cur = build_ffn(cur,377        model.layers[il].ffn_up, NULL, model.layers[il].ffn_up_s,378        model.layers[il].ffn_gate, NULL, model.layers[il].ffn_gate_s,379        model.layers[il].ffn_down, NULL, model.layers[il].ffn_down_s,380        NULL,381        LLM_FFN_SILU, LLM_FFN_PAR, il);382    cb(cur, "ffn_out", il);383 384    return cur;385}386