ReactiveAI/RxT-Beta-Decoder-SMAT

1{2  "att_groups": 4,3  "att_heads": 16,4  "att_query_groups": 8,5  "cross_att_type": "sqa",6  "dense_layer_dim": 1536,7  "embed_dim": 512,8  "ff_activation": "silu",9  "ff_dim": 192,10  "ff_dropout": 0.0,11  "final_stateless_layers_config": [12    "moe",13    "moe"14  ],15  "head_norm_type": "rms_norm",16  "moe_bias_mode": "global",17  "moe_grouped_gemm": true,18  "moe_shared_experts_bias_mode": "global",19  "moe_top_k": 10,20  "moe_use_cutlass_grouped_gemm": true,21  "moe_use_weighted_shared_experts": false,22  "num_experts": 384,23  "num_layers": 21,24  "num_shared_experts": 2,25  "rope_base": 100000,26  "router_amp": true,27  "router_dtype": "bfloat16",28  "self_att_type": "sqa",29  "seq_len": 8192,30  "shared_expert_dim": 384,31  "stateless_layers_config": [32    "dense",33    "moe"34  ],35  "stm_size": 4096,36  "use_attention_output_bias": false,37  "use_flash_attention": true,38  "use_gated": true,39  "use_gated_attention": true,40  "use_gated_cross_attention": false,41  "use_head_norm": true,42  "use_moe": true,43  "use_vectorized_moe": true,44  "vocab_size": 6553645}