codefuse-ai/CodeFuse-DevOps-Model-7B-Base

1# Copyright (c) Alibaba Cloud.2#3# This source code is licensed under the license found in the4# LICENSE file in the root directory of this source tree.5 6from transformers import PretrainedConfig7 8 9class QWenConfig(PretrainedConfig):10    model_type = "qwen"11    keys_to_ignore_at_inference = ["past_key_values"]12    attribute_map = {13        "hidden_size": "n_embd",14        "num_attention_heads": "n_head",15        "max_position_embeddings": "n_positions",16        "num_hidden_layers": "n_layer",17    }18 19    def __init__(20        self,21        vocab_size=151851,22        n_embd=4096,23        n_layer=32,24        n_head=32,25        n_inner=None,26        embd_pdrop=0.0,27        attn_pdrop=0.0,28        layer_norm_epsilon=1e-5,29        initializer_range=0.02,30        scale_attn_weights=True,31        use_cache=True,32        eos_token_id=151643,33        apply_residual_connection_post_layernorm=False,34        bf16=False,35        fp16=False,36        fp32=False,37        kv_channels=128,38        rotary_pct=1.0,39        rotary_emb_base=10000,40        use_dynamic_ntk=False,41        use_logn_attn=False,42        use_flash_attn=True,43        ffn_hidden_size=22016,44        no_bias=True,45        tie_word_embeddings=False,46        **kwargs,47    ):48        self.eos_token_id = eos_token_id49        super().__init__(50            eos_token_id=eos_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs51        )52 53        self.vocab_size = vocab_size54        self.n_embd = n_embd55        self.n_layer = n_layer56        self.n_head = n_head57        self.n_inner = n_inner58        self.embd_pdrop = embd_pdrop59        self.attn_pdrop = attn_pdrop60        self.layer_norm_epsilon = layer_norm_epsilon61        self.initializer_range = initializer_range62        self.scale_attn_weights = scale_attn_weights63        self.use_cache = use_cache64        self.apply_residual_connection_post_layernorm = (65            apply_residual_connection_post_layernorm66        )67        self.bf16 = bf1668        self.fp16 = fp1669        self.fp32 = fp3270        self.kv_channels = kv_channels71        self.rotary_pct = rotary_pct72        self.rotary_emb_base = rotary_emb_base73        self.use_dynamic_ntk = use_dynamic_ntk74        self.use_logn_attn = use_logn_attn75        self.use_flash_attn = use_flash_attn76        self.ffn_hidden_size = ffn_hidden_size77        self.no_bias = no_bias78        self.tie_word_embeddings = tie_word_embeddings79