OsaurusAI/MiniMax-M3-Coder-Small
6299
1{2 "architectures": [3 "MiniMaxM3SparseForConditionalGeneration"4 ],5 "auto_map": {6 "AutoConfig": "configuration_minimax_m3_vl.MiniMaxM3VLConfig"7 },8 "model_type": "minimax_m3_vl",9 "text_config": {10 "hidden_size": 6144,11 "intermediate_size": 3072,12 "num_hidden_layers": 60,13 "num_attention_heads": 64,14 "num_key_value_heads": 4,15 "head_dim": 128,16 "vocab_size": 200064,17 "max_position_embeddings": 1048576,18 "rms_norm_eps": 1e-06,19 "use_gemma_norm": true,20 "attention_output_gate": false,21 "rope_theta": 5000000,22 "rotary_dim": 64,23 "partial_rotary_factor": 0.5,24 "hidden_act": "swigluoai",25 "use_qk_norm": true,26 "tie_word_embeddings": false,27 "dense_intermediate_size": 12288,28 "shared_intermediate_size": 3072,29 "num_local_experts": 70,30 "num_experts_per_tok": 4,31 "n_shared_experts": 1,32 "scoring_func": "sigmoid",33 "use_routing_bias": true,34 "moe_layer_freq": [35 0,36 0,37 0,38 1,39 1,40 1,41 1,42 1,43 1,44 1,45 1,46 1,47 1,48 1,49 1,50 1,51 1,52 1,53 1,54 1,55 1,56 1,57 1,58 1,59 1,60 1,61 1,62 1,63 1,64 1,65 1,66 1,67 1,68 1,69 1,70 1,71 1,72 1,73 1,74 1,75 1,76 1,77 1,78 1,79 1,80 1,81 1,82 1,83 1,84 1,85 1,86 1,87 1,88 1,89 1,90 1,91 1,92 1,93 1,94 195 ],96 "qk_norm_type": "per_head",97 "num_mtp_modules": 7,98 "num_nextn_predict_layers": 1,99 "swiglu_alpha": 1.702,100 "swiglu_limit": 7.0,101 "routed_scaling_factor": 2.0,102 "sparse_attention_config": {103 "use_sparse_attention": true,104 "sparse_index_dim": 128,105 "sparse_num_index_heads": 4,106 "sparse_topk_blocks": 16,107 "sparse_block_size": 128,108 "sparse_disable_index_value": [109 0,110 0,111 0,112 1,113 1,114 1,115 1,116 1,117 1,118 1,119 1,120 1,121 1,122 1,123 1,124 1,125 1,126 1,127 1,128 1,129 1,130 1,131 1,132 1,133 1,134 1,135 1,136 1,137 1,138 1,139 1,140 1,141 1,142 1,143 1,144 1,145 1,146 1,147 1,148 1,149 1,150 1,151 1,152 1,153 1,154 1,155 1,156 1,157 1,158 1,159 1,160 1,161 1,162 1,163 1,164 1,165 1,166 1,167 1,168 1169 ],170 "sparse_score_type": "max",171 "sparse_init_block": 0,172 "sparse_local_block": 1,173 "sparse_attention_freq": [174 0,175 0,176 0,177 1,178 1,179 1,180 1,181 1,182 1,183 1,184 1,185 1,186 1,187 1,188 1,189 1,190 1,191 1,192 1,193 1,194 1,195 1,196 1,197 1,198 1,199 1,200 1,201 1,202 1,203 1,204 1,205 1,206 1,207 1,208 1,209 1,210 1,211 1,212 1,213 1,214 1,215 1,216 1,217 1,218 1,219 1,220 1,221 1,222 1,223 1,224 1,225 1,226 1,227 1,228 1,229 1,230 1,231 1,232 1,233 1234 ]235 },236 "architectures": [237 "MiniMaxM3SparseForCausalLM"238 ]239 },240 "vision_config": {241 "hidden_size": 1280,242 "num_attention_heads": 16,243 "num_hidden_layers": 32,244 "intermediate_size": 5120,245 "patch_size": 14,246 "image_size": 2016,247 "projection_dim": 6144,248 "position_embedding_type": "rope",249 "rope_mode": "3d",250 "rope_theta": 10000.0,251 "attention_dropout": 0.0,252 "hidden_act": "gelu",253 "initializer_factor": 1.0,254 "initializer_range": 0.02,255 "layer_norm_eps": 1e-05,256 "model_type": "clip_vision_model",257 "num_channels": 3,258 "vocab_size": 32000,259 "img_token_compression_config": {260 "image_token_compression_method": "patch_merge",261 "spatial_merge_size": 2,262 "temporal_patch_size": 2263 },264 "vision_segment_max_frames": 4265 },266 "img_token_compression_config": {267 "image_token_compression_method": "patch_merge",268 "spatial_merge_size": 2,269 "temporal_patch_size": 2270 },271 "image_grid_pinpoints": "[(336, 336), (336, 672), (336, 1008), (336, 1344), (336, 1680), (336, 2016), (672, 336), (672, 672), (672, 1008), (672, 1344), (672, 1680), (672, 2016), (1008, 336), (1008, 672), (1008, 1008), (1008, 1344), (1008, 1680), (1008, 2016), (1344, 336), (1344, 672), (1344, 1008), (1344, 1344), (1344, 1680), (1344, 2016), (1680, 336), (1680, 672), (1680, 1008), (1680, 1344), (1680, 1680), (1680, 2016), (2016, 336), (2016, 672), (2016, 1008), (2016, 1344), (2016, 1680), (2016, 2016)]",272 "image_seq_length": 576,273 "image_token_index": 200025,274 "video_token_index": 200026,275 "multimodal_projector_bias": true,276 "num_reward_heads": 0,277 "process_image_mode": "dynamic_res",278 "projector_hidden_act": "gelu",279 "vision_feature_layer": -1,280 "vision_feature_select_strategy": "full",281 "torch_dtype": "bfloat16",282 "transformers_version": "4.52.4",283 "projector_hidden_size": 6144,284 "quantization": {285 "bits": 8,286 "group_size": 64,287 "mode": "affine",288 "language_model.model.embed_tokens": {289 "bits": 6,290 "group_size": 64,291 "mode": "affine"292 },293 "language_model.lm_head": {294 "bits": 8,295 "group_size": 64,296 "mode": "affine"297 },298 "language_model.model.layers.0.self_attn.q_proj": {299 "bits": 8,300 "group_size": 64,301 "mode": "affine"302 },303 "language_model.model.layers.0.self_attn.k_proj": {304 "bits": 8,305 "group_size": 64,306 "mode": "affine"307 },308 "language_model.model.layers.0.self_attn.v_proj": {309 "bits": 8,310 "group_size": 64,311 "mode": "affine"312 },313 "language_model.model.layers.0.self_attn.o_proj": {314 "bits": 8,315 "group_size": 64,316 "mode": "affine"317 },318 "language_model.model.layers.0.mlp.gate_proj": {319 "bits": 6,320 "group_size": 64,321 "mode": "affine"322 },323 "language_model.model.layers.0.mlp.up_proj": {324 "bits": 6,325 "group_size": 64,326 "mode": "affine"327 },328 "language_model.model.layers.0.mlp.down_proj": {329 "bits": 6,330 "group_size": 64,331 "mode": "affine"332 },333 "language_model.model.layers.1.self_attn.q_proj": {334 "bits": 8,335 "group_size": 64,336 "mode": "affine"337 },338 "language_model.model.layers.1.self_attn.k_proj": {339 "bits": 8,340 "group_size": 64,341 "mode": "affine"342 },343 "language_model.model.layers.1.self_attn.v_proj": {344 "bits": 8,345 "group_size": 64,346 "mode": "affine"347 },348 "language_model.model.layers.1.self_attn.o_proj": {349 "bits": 8,350 "group_size": 64,351 "mode": "affine"352 },353 "language_model.model.layers.1.mlp.gate_proj": {354 "bits": 6,355 "group_size": 64,356 "mode": "affine"357 },358 "language_model.model.layers.1.mlp.up_proj": {359 "bits": 6,360 "group_size": 64,361 "mode": "affine"362 },363 "language_model.model.layers.1.mlp.down_proj": {364 "bits": 6,365 "group_size": 64,366 "mode": "affine"367 },368 "language_model.model.layers.2.self_attn.q_proj": {369 "bits": 8,370 "group_size": 64,371 "mode": "affine"372 },373 "language_model.model.layers.2.self_attn.k_proj": {374 "bits": 8,375 "group_size": 64,376 "mode": "affine"377 },378 "language_model.model.layers.2.self_attn.v_proj": {379 "bits": 8,380 "group_size": 64,381 "mode": "affine"382 },383 "language_model.model.layers.2.self_attn.o_proj": {384 "bits": 8,385 "group_size": 64,386 "mode": "affine"387 },388 "language_model.model.layers.2.mlp.gate_proj": {389 "bits": 6,390 "group_size": 64,391 "mode": "affine"392 },393 "language_model.model.layers.2.mlp.up_proj": {394 "bits": 6,395 "group_size": 64,396 "mode": "affine"397 },398 "language_model.model.layers.2.mlp.down_proj": {399 "bits": 6,400 "group_size": 64,401 "mode": "affine"402 },403 "language_model.model.layers.3.self_attn.q_proj": {404 "bits": 8,405 "group_size": 64,406 "mode": "affine"407 },408 "language_model.model.layers.3.self_attn.k_proj": {409 "bits": 8,410 "group_size": 64,411 "mode": "affine"412 },413 "language_model.model.layers.3.self_attn.v_proj": {414 "bits": 8,415 "group_size": 64,416 "mode": "affine"417 },418 "language_model.model.layers.3.self_attn.o_proj": {419 "bits": 8,420 "group_size": 64,421 "mode": "affine"422 },423 "language_model.model.layers.3.block_sparse_moe.shared_experts.gate_proj": {424 "bits": 6,425 "group_size": 64,426 "mode": "affine"427 },428 "language_model.model.layers.3.block_sparse_moe.shared_experts.up_proj": {429 "bits": 6,430 "group_size": 64,431 "mode": "affine"432 },433 "language_model.model.layers.3.block_sparse_moe.shared_experts.down_proj": {434 "bits": 6,435 "group_size": 64,436 "mode": "affine"437 },438 "language_model.model.layers.3.block_sparse_moe.switch_mlp.gate_proj": {439 "bits": 2,440 "group_size": 64,441 "mode": "affine"442 },443 "language_model.model.layers.3.block_sparse_moe.switch_mlp.up_proj": {444 "bits": 2,445 "group_size": 64,446 "mode": "affine"447 },448 "language_model.model.layers.3.block_sparse_moe.switch_mlp.down_proj": {449 "bits": 2,450 "group_size": 64,451 "mode": "affine"452 },453 "language_model.model.layers.4.self_attn.q_proj": {454 "bits": 8,455 "group_size": 64,456 "mode": "affine"457 },458 "language_model.model.layers.4.self_attn.k_proj": {459 "bits": 8,460 "group_size": 64,461 "mode": "affine"462 },463 "language_model.model.layers.4.self_attn.v_proj": {464 "bits": 8,465 "group_size": 64,466 "mode": "affine"467 },468 "language_model.model.layers.4.self_attn.o_proj": {469 "bits": 8,470 "group_size": 64,471 "mode": "affine"472 },473 "language_model.model.layers.4.block_sparse_moe.shared_experts.gate_proj": {474 "bits": 6,475 "group_size": 64,476 "mode": "affine"477 },478 "language_model.model.layers.4.block_sparse_moe.shared_experts.up_proj": {479 "bits": 6,480 "group_size": 64,481 "mode": "affine"482 },483 "language_model.model.layers.4.block_sparse_moe.shared_experts.down_proj": {484 "bits": 6,485 "group_size": 64,486 "mode": "affine"487 },488 "language_model.model.layers.4.block_sparse_moe.switch_mlp.gate_proj": {489 "bits": 2,490 "group_size": 64,491 "mode": "affine"492 },493 "language_model.model.layers.4.block_sparse_moe.switch_mlp.up_proj": {494 "bits": 2,495 "group_size": 64,496 "mode": "affine"497 },498 "language_model.model.layers.4.block_sparse_moe.switch_mlp.down_proj": {499 "bits": 2,500 "group_size": 64,501 "mode": "affine"502 },503 "language_model.model.layers.5.self_attn.q_proj": {504 "bits": 8,505 "group_size": 64,506 "mode": "affine"507 },508 "language_model.model.layers.5.self_attn.k_proj": {509 "bits": 8,510 "group_size": 64,511 "mode": "affine"512 },513 "language_model.model.layers.5.self_attn.v_proj": {514 "bits": 8,515 "group_size": 64,516 "mode": "affine"517 },518 "language_model.model.layers.5.self_attn.o_proj": {519 "bits": 8,520 "group_size": 64,521 "mode": "affine"522 },523 "language_model.model.layers.5.block_sparse_moe.shared_experts.gate_proj": {524 "bits": 6,525 "group_size": 64,526 "mode": "affine"527 },528 "language_model.model.layers.5.block_sparse_moe.shared_experts.up_proj": {529 "bits": 6,530 "group_size": 64,531 "mode": "affine"532 },533 "language_model.model.layers.5.block_sparse_moe.shared_experts.down_proj": {534 "bits": 6,535 "group_size": 64,536 "mode": "affine"537 },538 "language_model.model.layers.5.block_sparse_moe.switch_mlp.gate_proj": {539 "bits": 2,540 "group_size": 64,541 "mode": "affine"542 },543 "language_model.model.layers.5.block_sparse_moe.switch_mlp.up_proj": {544 "bits": 2,545 "group_size": 64,546 "mode": "affine"547 },548 "language_model.model.layers.5.block_sparse_moe.switch_mlp.down_proj": {549 "bits": 2,550 "group_size": 64,551 "mode": "affine"552 },553 "language_model.model.layers.6.self_attn.q_proj": {554 "bits": 8,555 "group_size": 64,556 "mode": "affine"557 },558 "language_model.model.layers.6.self_attn.k_proj": {559 "bits": 8,560 "group_size": 64,561 "mode": "affine"562 },563 "language_model.model.layers.6.self_attn.v_proj": {564 "bits": 8,565 "group_size": 64,566 "mode": "affine"567 },568 "language_model.model.layers.6.self_attn.o_proj": {569 "bits": 8,570 "group_size": 64,571 "mode": "affine"572 },573 "language_model.model.layers.6.block_sparse_moe.shared_experts.gate_proj": {574 "bits": 6,575 "group_size": 64,576 "mode": "affine"577 },578 "language_model.model.layers.6.block_sparse_moe.shared_experts.up_proj": {579 "bits": 6,580 "group_size": 64,581 "mode": "affine"582 },583 "language_model.model.layers.6.block_sparse_moe.shared_experts.down_proj": {584 "bits": 6,585 "group_size": 64,586 "mode": "affine"587 },588 "language_model.model.layers.6.block_sparse_moe.switch_mlp.gate_proj": {589 "bits": 2,590 "group_size": 64,591 "mode": "affine"592 },593 "language_model.model.layers.6.block_sparse_moe.switch_mlp.up_proj": {594 "bits": 2,595 "group_size": 64,596 "mode": "affine"597 },598 "language_model.model.layers.6.block_sparse_moe.switch_mlp.down_proj": {599 "bits": 2,600 "group_size": 64,601 "mode": "affine"602 },603 "language_model.model.layers.7.self_attn.q_proj": {604 "bits": 8,605 "group_size": 64,606 "mode": "affine"607 },608 "language_model.model.layers.7.self_attn.k_proj": {609 "bits": 8,610 "group_size": 64,611 "mode": "affine"612 },613 "language_model.model.layers.7.self_attn.v_proj": {614 "bits": 8,615 "group_size": 64,616 "mode": "affine"617 },618 "language_model.model.layers.7.self_attn.o_proj": {619 "bits": 8,620 "group_size": 64,621 "mode": "affine"622 },623 "language_model.model.layers.7.block_sparse_moe.shared_experts.gate_proj": {624 "bits": 6,625 "group_size": 64,626 "mode": "affine"627 },628 "language_model.model.layers.7.block_sparse_moe.shared_experts.up_proj": {629 "bits": 6,630 "group_size": 64,631 "mode": "affine"632 },633 "language_model.model.layers.7.block_sparse_moe.shared_experts.down_proj": {634 "bits": 6,635 "group_size": 64,636 "mode": "affine"637 },638 "language_model.model.layers.7.block_sparse_moe.switch_mlp.gate_proj": {639 "bits": 2,640 "group_size": 64,641 "mode": "affine"642 },643 "language_model.model.layers.7.block_sparse_moe.switch_mlp.up_proj": {644 "bits": 2,645 "group_size": 64,646 "mode": "affine"647 },648 "language_model.model.layers.7.block_sparse_moe.switch_mlp.down_proj": {649 "bits": 2,650 "group_size": 64,651 "mode": "affine"652 },653 "language_model.model.layers.8.self_attn.q_proj": {654 "bits": 8,655 "group_size": 64,656 "mode": "affine"657 },658 "language_model.model.layers.8.self_attn.k_proj": {659 "bits": 8,660 "group_size": 64,661 "mode": "affine"662 },663 "language_model.model.layers.8.self_attn.v_proj": {664 "bits": 8,665 "group_size": 64,666 "mode": "affine"667 },668 "language_model.model.layers.8.self_attn.o_proj": {669 "bits": 8,670 "group_size": 64,671 "mode": "affine"672 },673 "language_model.model.layers.8.block_sparse_moe.shared_experts.gate_proj": {674 "bits": 6,675 "group_size": 64,676 "mode": "affine"677 },678 "language_model.model.layers.8.block_sparse_moe.shared_experts.up_proj": {679 "bits": 6,680 "group_size": 64,681 "mode": "affine"682 },683 "language_model.model.layers.8.block_sparse_moe.shared_experts.down_proj": {684 "bits": 6,685 "group_size": 64,686 "mode": "affine"687 },688 "language_model.model.layers.8.block_sparse_moe.switch_mlp.gate_proj": {689 "bits": 2,690 "group_size": 64,691 "mode": "affine"692 },693 "language_model.model.layers.8.block_sparse_moe.switch_mlp.up_proj": {694 "bits": 2,695 "group_size": 64,696 "mode": "affine"697 },698 "language_model.model.layers.8.block_sparse_moe.switch_mlp.down_proj": {699 "bits": 2,700 "group_size": 64,701 "mode": "affine"702 },703 "language_model.model.layers.9.self_attn.q_proj": {704 "bits": 8,705 "group_size": 64,706 "mode": "affine"707 },708 "language_model.model.layers.9.self_attn.k_proj": {709 "bits": 8,710 "group_size": 64,711 "mode": "affine"712 },713 "language_model.model.layers.9.self_attn.v_proj": {714 "bits": 8,715 "group_size": 64,716 "mode": "affine"717 },718 "language_model.model.layers.9.self_attn.o_proj": {719 "bits": 8,720 "group_size": 64,721 "mode": "affine"722 },723 "language_model.model.layers.9.block_sparse_moe.shared_experts.gate_proj": {724 "bits": 6,725 "group_size": 64,726 "mode": "affine"727 },728 "language_model.model.layers.9.block_sparse_moe.shared_experts.up_proj": {729 "bits": 6,730 "group_size": 64,731 "mode": "affine"732 },733 "language_model.model.layers.9.block_sparse_moe.shared_experts.down_proj": {734 "bits": 6,735 "group_size": 64,736 "mode": "affine"737 },738 "language_model.model.layers.9.block_sparse_moe.switch_mlp.gate_proj": {739 "bits": 2,740 "group_size": 64,741 "mode": "affine"742 },743 "language_model.model.layers.9.block_sparse_moe.switch_mlp.up_proj": {744 "bits": 2,745 "group_size": 64,746 "mode": "affine"747 },748 "language_model.model.layers.9.block_sparse_moe.switch_mlp.down_proj": {749 "bits": 2,750 "group_size": 64,751 "mode": "affine"752 },753 "language_model.model.layers.10.self_attn.q_proj": {754 "bits": 8,755 "group_size": 64,756 "mode": "affine"757 },758 "language_model.model.layers.10.self_attn.k_proj": {759 "bits": 8,760 "group_size": 64,761 "mode": "affine"762 },763 "language_model.model.layers.10.self_attn.v_proj": {764 "bits": 8,765 "group_size": 64,766 "mode": "affine"767 },768 "language_model.model.layers.10.self_attn.o_proj": {769 "bits": 8,770 "group_size": 64,771 "mode": "affine"772 },773 "language_model.model.layers.10.block_sparse_moe.shared_experts.gate_proj": {774 "bits": 6,775 "group_size": 64,776 "mode": "affine"777 },778 "language_model.model.layers.10.block_sparse_moe.shared_experts.up_proj": {779 "bits": 6,780 "group_size": 64,781 "mode": "affine"782 },783 "language_model.model.layers.10.block_sparse_moe.shared_experts.down_proj": {784 "bits": 6,785 "group_size": 64,786 "mode": "affine"787 },788 "language_model.model.layers.10.block_sparse_moe.switch_mlp.gate_proj": {789 "bits": 2,790 "group_size": 64,791 "mode": "affine"792 },793 "language_model.model.layers.10.block_sparse_moe.switch_mlp.up_proj": {794 "bits": 2,795 "group_size": 64,796 "mode": "affine"797 },798 "language_model.model.layers.10.block_sparse_moe.switch_mlp.down_proj": {799 "bits": 2,800 "group_size": 64,801 "mode": "affine"802 },803 "language_model.model.layers.11.self_attn.q_proj": {804 "bits": 8,805 "group_size": 64,806 "mode": "affine"807 },808 "language_model.model.layers.11.self_attn.k_proj": {809 "bits": 8,810 "group_size": 64,811 "mode": "affine"812 },813 "language_model.model.layers.11.self_attn.v_proj": {814 "bits": 8,815 "group_size": 64,816 "mode": "affine"817 },818 "language_model.model.layers.11.self_attn.o_proj": {819 "bits": 8,820 "group_size": 64,821 "mode": "affine"822 },823 "language_model.model.layers.11.block_sparse_moe.shared_experts.gate_proj": {824 "bits": 6,825 "group_size": 64,826 "mode": "affine"827 },828 "language_model.model.layers.11.block_sparse_moe.shared_experts.up_proj": {829 "bits": 6,830 "group_size": 64,831 "mode": "affine"832 },833 "language_model.model.layers.11.block_sparse_moe.shared_experts.down_proj": {834 "bits": 6,835 "group_size": 64,836 "mode": "affine"837 },838 "language_model.model.layers.11.block_sparse_moe.switch_mlp.gate_proj": {839 "bits": 2,840 "group_size": 64,841 "mode": "affine"842 },843 "language_model.model.layers.11.block_sparse_moe.switch_mlp.up_proj": {844 "bits": 2,845 "group_size": 64,846 "mode": "affine"847 },848 "language_model.model.layers.11.block_sparse_moe.switch_mlp.down_proj": {849 "bits": 2,850 "group_size": 64,851 "mode": "affine"852 },853 "language_model.model.layers.12.self_attn.q_proj": {854 "bits": 8,855 "group_size": 64,856 "mode": "affine"857 },858 "language_model.model.layers.12.self_attn.k_proj": {859 "bits": 8,860 "group_size": 64,861 "mode": "affine"862 },863 "language_model.model.layers.12.self_attn.v_proj": {864 "bits": 8,865 "group_size": 64,866 "mode": "affine"867 },868 "language_model.model.layers.12.self_attn.o_proj": {869 "bits": 8,870 "group_size": 64,871 "mode": "affine"872 },873 "language_model.model.layers.12.block_sparse_moe.shared_experts.gate_proj": {874 "bits": 6,875 "group_size": 64,876 "mode": "affine"877 },878 "language_model.model.layers.12.block_sparse_moe.shared_experts.up_proj": {879 "bits": 6,880 "group_size": 64,881 "mode": "affine"882 },883 "language_model.model.layers.12.block_sparse_moe.shared_experts.down_proj": {884 "bits": 6,885 "group_size": 64,886 "mode": "affine"887 },888 "language_model.model.layers.12.block_sparse_moe.switch_mlp.gate_proj": {889 "bits": 2,890 "group_size": 64,891 "mode": "affine"892 },893 "language_model.model.layers.12.block_sparse_moe.switch_mlp.up_proj": {894 "bits": 2,895 "group_size": 64,896 "mode": "affine"897 },898 "language_model.model.layers.12.block_sparse_moe.switch_mlp.down_proj": {899 "bits": 2,900 "group_size": 64,901 "mode": "affine"902 },903 "language_model.model.layers.13.self_attn.q_proj": {904 "bits": 8,905 "group_size": 64,906 "mode": "affine"907 },908 "language_model.model.layers.13.self_attn.k_proj": {909 "bits": 8,910 "group_size": 64,911 "mode": "affine"912 },913 "language_model.model.layers.13.self_attn.v_proj": {914 "bits": 8,915 "group_size": 64,916 "mode": "affine"917 },918 "language_model.model.layers.13.self_attn.o_proj": {919 "bits": 8,920 "group_size": 64,921 "mode": "affine"922 },923 "language_model.model.layers.13.block_sparse_moe.shared_experts.gate_proj": {924 "bits": 6,925 "group_size": 64,926 "mode": "affine"927 },928 "language_model.model.layers.13.block_sparse_moe.shared_experts.up_proj": {929 "bits": 6,930 "group_size": 64,931 "mode": "affine"932 },933 "language_model.model.layers.13.block_sparse_moe.shared_experts.down_proj": {934 "bits": 6,935 "group_size": 64,936 "mode": "affine"937 },938 "language_model.model.layers.13.block_sparse_moe.switch_mlp.gate_proj": {939 "bits": 2,940 "group_size": 64,941 "mode": "affine"942 },943 "language_model.model.layers.13.block_sparse_moe.switch_mlp.up_proj": {944 "bits": 2,945 "group_size": 64,946 "mode": "affine"947 },948 "language_model.model.layers.13.block_sparse_moe.switch_mlp.down_proj": {949 "bits": 2,950 "group_size": 64,951 "mode": "affine"952 },953 "language_model.model.layers.14.self_attn.q_proj": {954 "bits": 8,955 "group_size": 64,956 "mode": "affine"957 },958 "language_model.model.layers.14.self_attn.k_proj": {959 "bits": 8,960 "group_size": 64,961 "mode": "affine"962 },963 "language_model.model.layers.14.self_attn.v_proj": {964 "bits": 8,965 "group_size": 64,966 "mode": "affine"967 },968 "language_model.model.layers.14.self_attn.o_proj": {969 "bits": 8,970 "group_size": 64,971 "mode": "affine"972 },973 "language_model.model.layers.14.block_sparse_moe.shared_experts.gate_proj": {974 "bits": 6,975 "group_size": 64,976 "mode": "affine"977 },978 "language_model.model.layers.14.block_sparse_moe.shared_experts.up_proj": {979 "bits": 6,980 "group_size": 64,981 "mode": "affine"982 },983 "language_model.model.layers.14.block_sparse_moe.shared_experts.down_proj": {984 "bits": 6,985 "group_size": 64,986 "mode": "affine"987 },988 "language_model.model.layers.14.block_sparse_moe.switch_mlp.gate_proj": {989 "bits": 2,990 "group_size": 64,991 "mode": "affine"992 },993 "language_model.model.layers.14.block_sparse_moe.switch_mlp.up_proj": {994 "bits": 2,995 "group_size": 64,996 "mode": "affine"997 },998 "language_model.model.layers.14.block_sparse_moe.switch_mlp.down_proj": {999 "bits": 2,1000 "group_size": 64,1001 "mode": "affine"1002 },1003 "language_model.model.layers.15.self_attn.q_proj": {1004 "bits": 8,1005 "group_size": 64,1006 "mode": "affine"1007 },1008 "language_model.model.layers.15.self_attn.k_proj": {1009 "bits": 8,1010 "group_size": 64,1011 "mode": "affine"1012 },1013 "language_model.model.layers.15.self_attn.v_proj": {1014 "bits": 8,1015 "group_size": 64,1016 "mode": "affine"1017 },1018 "language_model.model.layers.15.self_attn.o_proj": {1019 "bits": 8,1020 "group_size": 64,1021 "mode": "affine"1022 },1023 "language_model.model.layers.15.block_sparse_moe.shared_experts.gate_proj": {1024 "bits": 6,1025 "group_size": 64,1026 "mode": "affine"1027 },1028 "language_model.model.layers.15.block_sparse_moe.shared_experts.up_proj": {1029 "bits": 6,1030 "group_size": 64,1031 "mode": "affine"1032 },1033 "language_model.model.layers.15.block_sparse_moe.shared_experts.down_proj": {1034 "bits": 6,1035 "group_size": 64,1036 "mode": "affine"1037 },1038 "language_model.model.layers.15.block_sparse_moe.switch_mlp.gate_proj": {1039 "bits": 2,1040 "group_size": 64,1041 "mode": "affine"1042 },1043 "language_model.model.layers.15.block_sparse_moe.switch_mlp.up_proj": {1044 "bits": 2,1045 "group_size": 64,1046 "mode": "affine"1047 },1048 "language_model.model.layers.15.block_sparse_moe.switch_mlp.down_proj": {1049 "bits": 2,1050 "group_size": 64,1051 "mode": "affine"1052 },1053 "language_model.model.layers.16.self_attn.q_proj": {1054 "bits": 8,1055 "group_size": 64,1056 "mode": "affine"1057 },1058 "language_model.model.layers.16.self_attn.k_proj": {1059 "bits": 8,1060 "group_size": 64,1061 "mode": "affine"1062 },1063 "language_model.model.layers.16.self_attn.v_proj": {1064 "bits": 8,1065 "group_size": 64,1066 "mode": "affine"1067 },1068 "language_model.model.layers.16.self_attn.o_proj": {1069 "bits": 8,1070 "group_size": 64,1071 "mode": "affine"1072 },1073 "language_model.model.layers.16.block_sparse_moe.shared_experts.gate_proj": {1074 "bits": 6,1075 "group_size": 64,1076 "mode": "affine"1077 },1078 "language_model.model.layers.16.block_sparse_moe.shared_experts.up_proj": {1079 "bits": 6,1080 "group_size": 64,1081 "mode": "affine"1082 },1083 "language_model.model.layers.16.block_sparse_moe.shared_experts.down_proj": {1084 "bits": 6,1085 "group_size": 64,1086 "mode": "affine"1087 },1088 "language_model.model.layers.16.block_sparse_moe.switch_mlp.gate_proj": {1089 "bits": 2,1090 "group_size": 64,1091 "mode": "affine"1092 },1093 "language_model.model.layers.16.block_sparse_moe.switch_mlp.up_proj": {1094 "bits": 2,1095 "group_size": 64,1096 "mode": "affine"1097 },1098 "language_model.model.layers.16.block_sparse_moe.switch_mlp.down_proj": {1099 "bits": 2,1100 "group_size": 64,1101 "mode": "affine"1102 },1103 "language_model.model.layers.17.self_attn.q_proj": {1104 "bits": 8,1105 "group_size": 64,1106 "mode": "affine"1107 },1108 "language_model.model.layers.17.self_attn.k_proj": {1109 "bits": 8,1110 "group_size": 64,1111 "mode": "affine"1112 },1113 "language_model.model.layers.17.self_attn.v_proj": {1114 "bits": 8,1115 "group_size": 64,1116 "mode": "affine"1117 },1118 "language_model.model.layers.17.self_attn.o_proj": {1119 "bits": 8,1120 "group_size": 64,1121 "mode": "affine"1122 },1123 "language_model.model.layers.17.block_sparse_moe.shared_experts.gate_proj": {1124 "bits": 6,1125 "group_size": 64,1126 "mode": "affine"1127 },1128 "language_model.model.layers.17.block_sparse_moe.shared_experts.up_proj": {1129 "bits": 6,1130 "group_size": 64,1131 "mode": "affine"1132 },1133 "language_model.model.layers.17.block_sparse_moe.shared_experts.down_proj": {1134 "bits": 6,1135 "group_size": 64,1136 "mode": "affine"1137 },1138 "language_model.model.layers.17.block_sparse_moe.switch_mlp.gate_proj": {1139 "bits": 2,1140 "group_size": 64,1141 "mode": "affine"1142 },1143 "language_model.model.layers.17.block_sparse_moe.switch_mlp.up_proj": {1144 "bits": 2,1145 "group_size": 64,1146 "mode": "affine"1147 },1148 "language_model.model.layers.17.block_sparse_moe.switch_mlp.down_proj": {1149 "bits": 2,1150 "group_size": 64,1151 "mode": "affine"1152 },1153 "language_model.model.layers.18.self_attn.q_proj": {1154 "bits": 8,1155 "group_size": 64,1156 "mode": "affine"1157 },1158 "language_model.model.layers.18.self_attn.k_proj": {1159 "bits": 8,1160 "group_size": 64,1161 "mode": "affine"1162 },1163 "language_model.model.layers.18.self_attn.v_proj": {1164 "bits": 8,1165 "group_size": 64,1166 "mode": "affine"1167 },1168 "language_model.model.layers.18.self_attn.o_proj": {1169 "bits": 8,1170 "group_size": 64,1171 "mode": "affine"1172 },1173 "language_model.model.layers.18.block_sparse_moe.shared_experts.gate_proj": {1174 "bits": 6,1175 "group_size": 64,1176 "mode": "affine"1177 },1178 "language_model.model.layers.18.block_sparse_moe.shared_experts.up_proj": {1179 "bits": 6,1180 "group_size": 64,1181 "mode": "affine"1182 },1183 "language_model.model.layers.18.block_sparse_moe.shared_experts.down_proj": {1184 "bits": 6,1185 "group_size": 64,1186 "mode": "affine"1187 },1188 "language_model.model.layers.18.block_sparse_moe.switch_mlp.gate_proj": {1189 "bits": 2,1190 "group_size": 64,1191 "mode": "affine"1192 },1193 "language_model.model.layers.18.block_sparse_moe.switch_mlp.up_proj": {1194 "bits": 2,1195 "group_size": 64,1196 "mode": "affine"1197 },1198 "language_model.model.layers.18.block_sparse_moe.switch_mlp.down_proj": {1199 "bits": 2,1200 "group_size": 64,