codefuse-ai/CodeFuse-DevOps-Model-7B-Base

1{2  "activation": "swiglu",3  "apply_residual_connection_post_layernorm": false,4  "architectures": [5    "QWenLMHeadModel"6  ],7  "attn_pdrop": 0.0,8  "auto_map": {9    "AutoConfig": "configuration_qwen.QWenConfig",10    "AutoModel": "modeling_qwen.QWenLMHeadModel",11    "AutoModelForCausalLM": "modeling_qwen.QWenLMHeadModel"12  },13  "bf16": true,14  "bias_dropout_fusion": true,15  "bos_token_id": 151643,16  "embd_pdrop": 0.0,17  "eos_token_id": 151643,18  "ffn_hidden_size": 22016,19  "fp16": false,20  "fp32": false,21  "initializer_range": 0.02,22  "kv_channels": 128,23  "layer_norm_epsilon": 1e-06,24  "model_type": "qwen",25  "n_embd": 4096,26  "n_head": 32,27  "n_inner": null,28  "n_layer": 32,29  "n_positions": 6144,30  "no_bias": true,31  "onnx_safe": null,32  "padded_vocab_size": 151936,33  "params_dtype": "torch.bfloat16",34  "pos_emb": "rotary",35  "resid_pdrop": 0.1,36  "rotary_emb_base": 10000,37  "rotary_pct": 1.0,38  "scale_attn_weights": true,39  "seq_length": 2048,40  "tie_word_embeddings": false,41  "tokenizer_type": "QWenTokenizer",42  "torch_dtype": "bfloat16",43  "transformers_version": "4.32.0",44  "use_cache": true,45  "use_dynamic_ntk": true,46  "use_flash_attn": true,47  "use_logn_attn": true,48  "vocab_size": 15193649}50