Multilingual-Multimodal-NLP/LoopCoder-V2

1"""IQuestPLTCoder model configuration.2 3Extends the IQuestCoder configuration with PLT (Parallel Loop Transformer)4specific parameters. PLT reuses the same physical transformer layers across5multiple loops, with cross-loop processing (CLP) and mixed attention (global6full-attention + local sliding-window attention gated per head) in loop 1+.7 8Reference: https://arxiv.org/abs/2510.248249"""10 11from typing import Dict, List, Optional, Union12 13from transformers.configuration_utils import PretrainedConfig14from transformers.utils import logging15 16 17logger = logging.get_logger(__name__)18 19 20class IQuestPLTCoderConfig(PretrainedConfig):21    r"""22    Configuration class for [`IQuestPLTCoderModel`].23 24    This is a PLT (Parallel Loop Transformer) variant of IQuestCoder. The model25    has `num_hidden_layers` physical transformer layers that are executed26    `plt_num_loops` times. Weights are shared across loops; each loop adds27    cross-loop processing and mixed attention via a learned per-head gate.28 29    Args:30        vocab_size (`int`, *optional*, defaults to 75904):31            Vocabulary size of the model (padded to be divisible by 128).32        hidden_size (`int`, *optional*, defaults to 5120):33            Dimension of the hidden representations.34        intermediate_size (`int`, *optional*, defaults to 27648):35            Dimension of the MLP representations.36        num_hidden_layers (`int`, *optional*, defaults to 14):37            Number of physical transformer layers (shared across all loops).38        num_attention_heads (`int`, *optional*, defaults to 40):39            Number of attention heads for each attention layer.40        num_key_value_heads (`int`, *optional*, defaults to 8):41            Number of key_value heads for Grouped Query Attention (GQA).42        head_dim (`int`, *optional*, defaults to 128):43            The dimension of each attention head.44        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):45            The non-linear activation function in the decoder (SwiGLU uses SiLU).46        max_position_embeddings (`int`, *optional*, defaults to 131072):47            The maximum sequence length that this model might ever be used with.48        initializer_range (`float`, *optional*, defaults to 0.02):49            The standard deviation of the truncated_normal_initializer for50            initializing all weight matrices.51        rms_norm_eps (`float`, *optional*, defaults to 1e-05):52            The epsilon used by the RMS normalization layers.53        use_cache (`bool`, *optional*, defaults to `True`):54            Whether the model should return the last key/values attentions.55        pad_token_id (`int`, *optional*):56            Padding token id.57        bos_token_id (`int`, *optional*, defaults to 1):58            Beginning of stream token id.59        eos_token_id (`int` or `list`, *optional*, defaults to `[2, 75864, 75869]`):60            End of stream token id(s).61        tie_word_embeddings (`bool`, *optional*, defaults to `False`):62            Whether to tie input embedding and output projection weights.63        rope_theta (`float`, *optional*, defaults to 500000.0):64            The base period of the RoPE embeddings.65        rope_scaling (`Dict`, *optional*):66            Dictionary containing the scaling configuration for the RoPE67            embeddings. Supports "linear", "dynamic", "yarn", "longrope", "llama3".68        attention_bias (`bool`, *optional*, defaults to `False`):69            Whether to use a bias in the Q, K, V and output projection layers.70        attention_dropout (`float`, *optional*, defaults to 0.0):71            The dropout ratio for the attention probabilities.72        mlp_bias (`bool`, *optional*, defaults to `False`):73            Whether to use a bias in the MLP gate/up/down projection layers.74        plt_num_loops (`int`, *optional*, defaults to 2):75            Number of times the physical transformer layers are executed.76            Loop 0 runs standard causal attention and stores KV caches.77            Loops 1+ run mixed attention with cross-loop processing.78        plt_window_size (`list` of `int`, *optional*, defaults to `[64, 0]`):79            Sliding window size `[left, right]` for the local attention in80            loop 1+. `[64, 0]` means a left-context window of 64 tokens with81            causal masking (right=0).82        plt_normalize_per_loop (`bool`, *optional*, defaults to `True`):83            When True, apply final_layernorm (shared weights) to hidden states84            at the end of each non-last loop before cross-loop processing.85        plt_emb_scale (`float`, *optional*, defaults to `None`):86            Scaling factor for the embedding in CLP: `a * E + b * shift(H)`.87            `None` means 1.0 (no scaling).88        plt_hidden_scale (`float`, *optional*, defaults to `None`):89            Scaling factor for the shifted hidden state in CLP:90            `a * E + b * shift(H)`. `None` means 1.0 (no scaling).91        plt_gate_use_hidden_states (`bool`, *optional*, defaults to `False`):92            Gate input mode. When `False`, the gate is computed as93            `sigmoid(einsum(Q, W_gate) + b_gate)` per head on the post-RoPE94            query tensor.  When `True`, gate uses95            `sigmoid(Linear(RMSNorm(hidden_states)))` (OLMo-style) instead.96 97    Example:98        ```python99        >>> from configuration_iquestpltcoder import IQuestPLTCoderConfig100        >>> from modeling_iquestpltcoder import IQuestPLTCoderModel101 102        >>> configuration = IQuestPLTCoderConfig()103        >>> model = IQuestPLTCoderModel(configuration)104        >>> configuration = model.config105        ```106    """107 108    model_type = "iquestpltcoder"109    keys_to_ignore_at_inference = ["past_key_values"]110 111    def __init__(112        self,113        vocab_size=75904,114        hidden_size=5120,115        intermediate_size=27648,116        num_hidden_layers=14,117        num_attention_heads=40,118        num_key_value_heads=8,119        head_dim=128,120        hidden_act="silu",121        max_position_embeddings=131072,122        initializer_range=0.02,123        rms_norm_eps=1e-5,124        use_cache=True,125        pad_token_id=None,126        bos_token_id=1,127        eos_token_id=None,128        tie_word_embeddings=False,129        rope_theta=500000.0,130        rope_scaling=None,131        attention_bias=False,132        attention_dropout=0.0,133        mlp_bias=False,134        # PLT specific135        plt_num_loops=2,136        plt_window_size=None,137        plt_normalize_per_loop=True,138        plt_emb_scale=None,139        plt_hidden_scale=None,140        plt_gate_use_hidden_states=False,141        **kwargs,142    ):143        if eos_token_id is None:144            eos_token_id = [2, 75864, 75869]145        if plt_window_size is None:146            plt_window_size = [64, 0]147 148        self.vocab_size = vocab_size149        self.max_position_embeddings = max_position_embeddings150        self.hidden_size = hidden_size151        self.intermediate_size = intermediate_size152        self.num_hidden_layers = num_hidden_layers153        self.num_attention_heads = num_attention_heads154        self.num_key_value_heads = num_key_value_heads155        self.head_dim = head_dim156        self.hidden_act = hidden_act157        self.initializer_range = initializer_range158        self.rms_norm_eps = rms_norm_eps159        self.use_cache = use_cache160        self.rope_theta = rope_theta161        self.rope_scaling = rope_scaling162        self.attention_bias = attention_bias163        self.attention_dropout = attention_dropout164        self.mlp_bias = mlp_bias165 166        # PLT specific167        self.plt_num_loops = plt_num_loops168        self.plt_window_size = plt_window_size169        self.plt_normalize_per_loop = plt_normalize_per_loop170        self.plt_emb_scale = plt_emb_scale171        self.plt_hidden_scale = plt_hidden_scale172        self.plt_gate_use_hidden_states = plt_gate_use_hidden_states173 174        self._rope_scaling_validation()175 176        super().__init__(177            pad_token_id=pad_token_id,178            bos_token_id=bos_token_id,179            eos_token_id=eos_token_id,180            tie_word_embeddings=tie_word_embeddings,181            **kwargs,182        )183 184    def _rope_scaling_validation(self):185        """Validate the `rope_scaling` configuration."""186        if self.rope_scaling is None:187            return188 189        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) < 1:190            raise ValueError(191                "`rope_scaling` must be a dictionary with a minimum of one field, "192                "`type` or `rope_type`."193            )194 195        rope_scaling_type = self.rope_scaling.get("type", None) or self.rope_scaling.get(196            "rope_type", None197        )198        if rope_scaling_type is None:199            raise ValueError("`rope_scaling` must have a `type` or `rope_type` field.")200 201        valid_rope_types = ["linear", "dynamic", "yarn", "longrope", "llama3"]202        if rope_scaling_type not in valid_rope_types:203            raise ValueError(204                f"`rope_scaling`'s type field must be one of {valid_rope_types}, "205                f"got {rope_scaling_type}"206            )207 208 209__all__ = ["IQuestPLTCoderConfig"]210