Multilingual-Multimodal-NLP/LoopCoder-V2

1"""IQuestCoder model configuration."""2 3from transformers.configuration_utils import PretrainedConfig4from transformers.utils import logging5 6 7logger = logging.get_logger(__name__)8 9 10class IQuestCoderConfig(PretrainedConfig):11    r"""12    This is the configuration class to store the configuration of a [`IQuestCoderModel`]. It is used to instantiate13    an IQuestCoder model according to the specified arguments, defining the model architecture.14 15    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the16    documentation from [`PretrainedConfig`] for more information.17 18    Args:19        vocab_size (`int`, *optional*, defaults to 76800):20            Vocabulary size of the IQuestCoder model. Defines the number of different tokens that can be represented21            by the `inputs_ids` passed when calling [`IQuestCoderModel`].22        hidden_size (`int`, *optional*, defaults to 5120):23            Dimension of the hidden representations.24        intermediate_size (`int`, *optional*, defaults to 27648):25            Dimension of the MLP representations.26        num_hidden_layers (`int`, *optional*, defaults to 80):27            Number of hidden layers in the Transformer decoder.28        num_attention_heads (`int`, *optional*, defaults to 40):29            Number of attention heads for each attention layer in the Transformer decoder.30        num_key_value_heads (`int`, *optional*, defaults to 8):31            This is the number of key_value heads that should be used to implement Grouped Query Attention (GQA).32            If `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA).33            If `num_key_value_heads=1`, the model will use Multi Query Attention (MQA).34        head_dim (`int`, *optional*, defaults to 128):35            The dimension of each attention head. If not specified, defaults to `hidden_size // num_attention_heads`.36        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):37            The non-linear activation function (function or string) in the decoder.38        max_position_embeddings (`int`, *optional*, defaults to 16384):39            The maximum sequence length that this model might ever be used with.40        initializer_range (`float`, *optional*, defaults to 0.02):41            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.42        rms_norm_eps (`float`, *optional*, defaults to 1e-05):43            The epsilon used by the rms normalization layers.44        use_cache (`bool`, *optional*, defaults to `True`):45            Whether or not the model should return the last key/values attentions (not used by all models).46        pad_token_id (`int`, *optional*):47            Padding token id.48        bos_token_id (`int`, *optional*, defaults to 1):49            Beginning of stream token id.50        eos_token_id (`int`, *optional*, defaults to 2):51            End of stream token id.52        tie_word_embeddings (`bool`, *optional*, defaults to `False`):53            Whether to tie weight embeddings.54        rope_theta (`float`, *optional*, defaults to 500000.0):55            The base period of the RoPE embeddings.56        rope_scaling (`Dict`, *optional*):57            Dictionary containing the scaling configuration for the RoPE embeddings. Supports various RoPE scaling58            types including "linear", "dynamic", "yarn", "longrope", etc.59        attention_bias (`bool`, *optional*, defaults to `False`):60            Whether to use a bias in the query, key, value and output projection layers during self-attention.61        attention_dropout (`float`, *optional*, defaults to 0.0):62            The dropout ratio for the attention probabilities.63        mlp_bias (`bool`, *optional*, defaults to `False`):64            Whether to use a bias in up_proj, down_proj and gate_proj layers in the MLP layers.65        clip_qkv (`float`, *optional*):66            If set, clip the query, key, and value tensors to this value. Borrowed from OLMo for training stability.67        use_sliding_window (`bool`, *optional*, defaults to `False`):68            Whether to use sliding window attention. Borrowed from Qwen2.69        sliding_window (`int`, *optional*):70            The sliding window size. Only effective when `use_sliding_window=True`.71        max_window_layers (`int`, *optional*, defaults to 0):72            The number of layers that don't use sliding window attention. Borrowed from Qwen2.73 74    Example:75        ```python76        >>> from configuration_iquestcoder import IQuestCoderConfig77        >>> from modeling_iquestcoder import IQuestCoderModel78 79        >>> # Initializing a IQuestCoder configuration80        >>> configuration = IQuestCoderConfig()81 82        >>> # Initializing a model from the configuration83        >>> model = IQuestCoderModel(configuration)84 85        >>> # Accessing the model configuration86        >>> configuration = model.config87        ```88    """89 90    model_type = "iquestcoder"91    keys_to_ignore_at_inference = ["past_key_values"]92 93    # Tensor / pipeline parallel plans for vLLM transformers backend.94    # Same shape as LlamaConfig — IQuestCoder is structurally Llama (RMSNorm + GQA + RoPE + SwiGLU).95    base_model_tp_plan = {96        "layers.*.self_attn.q_proj": "colwise",97        "layers.*.self_attn.k_proj": "colwise",98        "layers.*.self_attn.v_proj": "colwise",99        "layers.*.self_attn.o_proj": "rowwise",100        "layers.*.mlp.gate_proj": "colwise",101        "layers.*.mlp.up_proj": "colwise",102        "layers.*.mlp.down_proj": "rowwise",103    }104    base_model_pp_plan = {105        "embed_tokens": (["input_ids"], ["inputs_embeds"]),106        "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),107        "norm": (["hidden_states"], ["hidden_states"]),108    }109 110    def __init__(111        self,112        vocab_size=76800,113        hidden_size=5120,114        intermediate_size=27648,115        num_hidden_layers=80,116        num_attention_heads=40,117        num_key_value_heads=8,118        head_dim=128,119        hidden_act="silu",120        max_position_embeddings=16384,121        initializer_range=0.02,122        rms_norm_eps=1e-5,123        use_cache=True,124        pad_token_id=None,125        bos_token_id=1,126        eos_token_id=2,127        tie_word_embeddings=False,128        rope_theta=500000.0,129        rope_scaling=None,130        attention_bias=False,131        attention_dropout=0.0,132        mlp_bias=False,133        # IQuestCoder specific (borrowed from OLMo)134        clip_qkv=None,135        # IQuestCoder specific (borrowed from Qwen2)136        use_sliding_window=False,137        sliding_window=None,138        max_window_layers=0,139        **kwargs,140    ):141        self.vocab_size = vocab_size142        self.max_position_embeddings = max_position_embeddings143        self.hidden_size = hidden_size144        self.intermediate_size = intermediate_size145        self.num_hidden_layers = num_hidden_layers146        self.num_attention_heads = num_attention_heads147        self.num_key_value_heads = num_key_value_heads148        self.head_dim = head_dim149        self.hidden_act = hidden_act150        self.initializer_range = initializer_range151        self.rms_norm_eps = rms_norm_eps152        self.use_cache = use_cache153        self.rope_theta = rope_theta154        self.rope_scaling = rope_scaling155        self.attention_bias = attention_bias156        self.attention_dropout = attention_dropout157        self.mlp_bias = mlp_bias158        # IQuestCoder specific159        self.clip_qkv = clip_qkv160        self.use_sliding_window = use_sliding_window161        self.sliding_window = sliding_window162        self.max_window_layers = max_window_layers163 164        # Validate rope_scaling configuration165        self._rope_scaling_validation()166 167        super().__init__(168            pad_token_id=pad_token_id,169            bos_token_id=bos_token_id,170            eos_token_id=eos_token_id,171            tie_word_embeddings=tie_word_embeddings,172            **kwargs,173        )174 175    def _rope_scaling_validation(self):176        """Validate the `rope_scaling` configuration."""177        if self.rope_scaling is None:178            return179 180        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) < 1:181            raise ValueError(182                "`rope_scaling` must be a dictionary with a minimum of one field, `type` or `rope_type`."183            )184        185        rope_scaling_type = self.rope_scaling.get("type", None) or self.rope_scaling.get("rope_type", None)186        if rope_scaling_type is None:187            raise ValueError(188                "`rope_scaling` must have a `type` or `rope_type` field."189            )190        191        valid_rope_types = ["linear", "dynamic", "yarn", "longrope", "llama3"]192        if rope_scaling_type not in valid_rope_types:193            raise ValueError(194                f"`rope_scaling`'s type field must be one of {valid_rope_types}, got {rope_scaling_type}"195            )196 197 198__all__ = ["IQuestCoderConfig"]199 200