xhyi/CodeGen-350M-Multi

1# coding=utf-82# Copyright 2021 The EleutherAI and HuggingFace Teams. All rights reserved.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16# Modified configuration implementation based on https://github.com/huggingface/transformers/blob/main/src/transformers/models/gptj/configuration_gptj.py17 18from transformers.configuration_utils import PretrainedConfig19from transformers.utils import logging20 21logger = logging.get_logger(__name__)22 23 24class CodeGenConfig(PretrainedConfig):25    model_type = "codegen"26 27    def __init__(28        self,29        vocab_size=50400,30        n_positions=2048,31        n_ctx=2048,32        n_embd=4096,33        n_layer=28,34        n_head=16,35        rotary_dim=64,36        n_inner=None,37        activation_function="gelu_new",38        resid_pdrop=0.0,39        embd_pdrop=0.0,40        attn_pdrop=0.0,41        layer_norm_epsilon=1e-5,42        initializer_range=0.02,43        scale_attn_weights=True,44        gradient_checkpointing=False,45        use_cache=True,46        bos_token_id=50256,47        eos_token_id=50256,48        **kwargs49    ):50        super().__init__(bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs)51 52        self.vocab_size = vocab_size53        self.n_ctx = n_ctx54        self.n_positions = n_positions55        self.n_embd = n_embd56        self.n_layer = n_layer57        self.n_head = n_head58        self.n_inner = n_inner59        self.rotary_dim = rotary_dim60        self.activation_function = activation_function61        self.resid_pdrop = resid_pdrop62        self.embd_pdrop = embd_pdrop63        self.attn_pdrop = attn_pdrop64        self.layer_norm_epsilon = layer_norm_epsilon65        self.initializer_range = initializer_range66        self.gradient_checkpointing = gradient_checkpointing67        self.scale_attn_weights = scale_attn_weights68        self.use_cache = use_cache69 70        self.bos_token_id = bos_token_id71        self.eos_token_id = eos_token_id72 73    @property74    def max_position_embeddings(self):75        return self.n_positions76 77    @property78    def hidden_size(self):79        return self.n_embd80 81    @property82    def num_attention_heads(self):83        return self.n_head84 85    @property86    def num_hidden_layers(self):87        return self.n_layer88