QuantTrio/GLM-5.2-Int4-Int8Mix
211.6k
1{2 "name_or_path": "tclf90/GLM-5.2-Int4-Int8Mix",3 "architectures": [4 "GlmMoeDsaForCausalLM"5 ],6 "attention_bias": false,7 "attention_dropout": 0.0,8 "dtype": "bfloat16",9 "eos_token_id": [10 154820,11 154827,12 15482913 ],14 "ep_size": 1,15 "first_k_dense_replace": 3,16 "head_dim": 192,17 "hidden_act": "silu",18 "hidden_size": 6144,19 "index_head_dim": 128,20 "index_n_heads": 32,21 "index_share_for_mtp_iteration": true,22 "index_skip_topk_offset": 3,23 "index_topk": 2048,24 "index_topk_freq": 4,25 "index_topk_pattern": null,26 "indexer_rope_interleave": true,27 "indexer_types": [28 "full",29 "full",30 "full",31 "shared",32 "shared",33 "shared",34 "full",35 "shared",36 "shared",37 "shared",38 "full",39 "shared",40 "shared",41 "shared",42 "full",43 "shared",44 "shared",45 "shared",46 "full",47 "shared",48 "shared",49 "shared",50 "full",51 "shared",52 "shared",53 "shared",54 "full",55 "shared",56 "shared",57 "shared",58 "full",59 "shared",60 "shared",61 "shared",62 "full",63 "shared",64 "shared",65 "shared",66 "full",67 "shared",68 "shared",69 "shared",70 "full",71 "shared",72 "shared",73 "shared",74 "full",75 "shared",76 "shared",77 "shared",78 "full",79 "shared",80 "shared",81 "shared",82 "full",83 "shared",84 "shared",85 "shared",86 "full",87 "shared",88 "shared",89 "shared",90 "full",91 "shared",92 "shared",93 "shared",94 "full",95 "shared",96 "shared",97 "shared",98 "full",99 "shared",100 "shared",101 "shared",102 "full",103 "shared",104 "shared",105 "shared"106 ],107 "initializer_range": 0.02,108 "intermediate_size": 12288,109 "kv_lora_rank": 512,110 "max_position_embeddings": 1048576,111 "mlp_layer_types": [112 "dense",113 "dense",114 "dense",115 "sparse",116 "sparse",117 "sparse",118 "sparse",119 "sparse",120 "sparse",121 "sparse",122 "sparse",123 "sparse",124 "sparse",125 "sparse",126 "sparse",127 "sparse",128 "sparse",129 "sparse",130 "sparse",131 "sparse",132 "sparse",133 "sparse",134 "sparse",135 "sparse",136 "sparse",137 "sparse",138 "sparse",139 "sparse",140 "sparse",141 "sparse",142 "sparse",143 "sparse",144 "sparse",145 "sparse",146 "sparse",147 "sparse",148 "sparse",149 "sparse",150 "sparse",151 "sparse",152 "sparse",153 "sparse",154 "sparse",155 "sparse",156 "sparse",157 "sparse",158 "sparse",159 "sparse",160 "sparse",161 "sparse",162 "sparse",163 "sparse",164 "sparse",165 "sparse",166 "sparse",167 "sparse",168 "sparse",169 "sparse",170 "sparse",171 "sparse",172 "sparse",173 "sparse",174 "sparse",175 "sparse",176 "sparse",177 "sparse",178 "sparse",179 "sparse",180 "sparse",181 "sparse",182 "sparse",183 "sparse",184 "sparse",185 "sparse",186 "sparse",187 "sparse",188 "sparse",189 "sparse"190 ],191 "model_type": "glm_moe_dsa",192 "moe_intermediate_size": 2048,193 "moe_layer_freq": 1,194 "n_group": 1,195 "n_routed_experts": 256,196 "n_shared_experts": 1,197 "norm_topk_prob": true,198 "num_attention_heads": 64,199 "num_experts_per_tok": 8,200 "num_hidden_layers": 78,201 "num_key_value_heads": 64,202 "num_nextn_predict_layers": 1,203 "pad_token_id": 154820,204 "pretraining_tp": 1,205 "q_lora_rank": 2048,206 "qk_head_dim": 256,207 "qk_nope_head_dim": 192,208 "qk_rope_head_dim": 64,209 "rms_norm_eps": 1e-05,210 "rope_interleave": true,211 "rope_parameters": {212 "rope_theta": 8000000,213 "rope_type": "default"214 },215 "routed_scaling_factor": 2.5,216 "scoring_func": "sigmoid",217 "tie_word_embeddings": false,218 "topk_group": 1,219 "topk_method": "noaux_tc",220 "transformers_version": "5.12.0",221 "use_cache": true,222 "v_head_dim": 256,223 "vocab_size": 154880,224 "quantization_config": {225 "quant_method": "compressed-tensors",226 "format": "pack-quantized",227 "ignore": [228 "re:model[.]layers[.]0[.].*",229 "re:model[.]layers[.][1-9][0-9]*[.](?:mtp_block[.])?mlp[.]gate(?:$|[.].*)",230 "re:model[.]layers[.][1-9][0-9]*[.](?:mtp_block[.])?self_attn[.]indexer(?:$|[.].*)",231 "re:model[.]layers[.][1-9][0-9]*[.](?:mtp_block[.])?self_attn[.]indexers_proj(?:$|[.].*)",232 "re:model[.]layers[.][1-9][0-9]*[.](?:eh_proj|enorm|hnorm)[.].*",233 "re:model[.]layers[.][1-9][0-9]*[.]shared_head[.]norm[.].*",234 "re:model[.]layers[.][1-9][0-9]*[.]shared_head[.]head(?:$|[.].*)"235 ],236 "config_groups": {237 "w4a16_experts": {238 "targets": [239 "re:model[.]layers[.](?:3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77)[.]mlp[.]experts[.][0-9]+[.](?:gate_proj|up_proj|down_proj)$"240 ],241 "weights": {242 "num_bits": 4,243 "type": "int",244 "symmetric": true,245 "strategy": "group",246 "group_size": 128,247 "dynamic": false248 }249 },250 "w8a16_linears": {251 "targets": [252 "re:model[.]layers[.](?:1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17|18|19|20|21|22|23|24|25|26|27|28|29|30|31|32|33|34|35|36|37|38|39|40|41|42|43|44|45|46|47|48|49|50|51|52|53|54|55|56|57|58|59|60|61|62|63|64|65|66|67|68|69|70|71|72|73|74|75|76|77)[.](?:self_attn[.](?:fused_qkv_a_proj_with_mqa|q_a_proj|q_b_proj|kv_a_proj_with_mqa|kv_b_proj|o_proj)|mlp[.](?:gate_up_proj|gate_proj|up_proj|down_proj|shared_experts[.](?:gate_up_proj|gate_proj|up_proj|down_proj)))$"253 ],254 "weights": {255 "num_bits": 8,256 "type": "int",257 "symmetric": true,258 "strategy": "group",259 "group_size": 128,260 "dynamic": false261 }262 },263 "w8a16_mtp_channel": {264 "targets": [265 "re:model[.]layers[.](?:78)[.](?:mtp_block[.])?(?:self_attn[.](?:fused_qkv_a_proj_with_mqa|q_a_proj|q_b_proj|kv_a_proj_with_mqa|kv_b_proj|o_proj)|mlp[.](?:experts[.][0-9]+[.](?:gate_proj|up_proj|down_proj)|gate_up_proj|gate_proj|up_proj|down_proj|shared_experts[.](?:gate_up_proj|gate_proj|up_proj|down_proj)))$"266 ],267 "weights": {268 "num_bits": 8,269 "type": "int",270 "symmetric": true,271 "strategy": "channel",272 "group_size": -1,273 "dynamic": false274 }275 }276 },277 "packed_modules_mapping": {278 "fused_qkv_a_proj_with_mqa": [279 "q_a_proj",280 "kv_a_proj_with_mqa"281 ],282 "gate_up_proj": [283 "gate_proj",284 "up_proj"285 ]286 }287 }288}289