AlgorithmicResearchGroup/phi-arxiv-cs-ml-instruct

1# Copyright (c) Microsoft Corporation.2# Licensed under the MIT license.3 4import math5from typing import Optional6 7from transformers import PretrainedConfig8 9 10class PhiConfig(PretrainedConfig):11    """Phi configuration."""12 13    model_type = "phi"14    attribute_map = {15        "max_position_embeddings": "n_positions",16        "hidden_size": "n_embd",17        "num_attention_heads": "n_head",18        "num_hidden_layers": "n_layer",19    }20 21    def __init__(22        self,23        vocab_size: int = 50304,24        n_positions: int = 2048,25        n_embd: int = 1024,26        n_layer: int = 20,27        n_inner: Optional[int] = None,28        n_head: int = 16,29        n_head_kv: Optional[int] = None,30        rotary_dim: Optional[int] = 32,31        activation_function: Optional[str] = "gelu_new",32        flash_attn: bool = False,33        flash_rotary: bool = False,34        fused_dense: bool = False,35        attn_pdrop: float = 0.0,36        embd_pdrop: float = 0.0,37        resid_pdrop: float = 0.0,38        layer_norm_epsilon: float = 1e-5,39        initializer_range: float = 0.02,40        tie_word_embeddings: bool = False,41        pad_vocab_size_multiple: int = 64,42        **kwargs43    ) -> None:44        self.vocab_size = int(math.ceil(vocab_size / pad_vocab_size_multiple) * pad_vocab_size_multiple)45        self.n_positions = n_positions46        self.n_embd = n_embd47        self.n_layer = n_layer48        self.n_inner = n_inner49        self.n_head = n_head50        self.n_head_kv = n_head_kv51        self.rotary_dim = min(rotary_dim, n_embd // n_head)52        self.activation_function = activation_function53        self.flash_attn = flash_attn54        self.flash_rotary = flash_rotary55        self.fused_dense = fused_dense56        self.attn_pdrop = attn_pdrop57        self.embd_pdrop = embd_pdrop58        self.resid_pdrop = resid_pdrop59        self.layer_norm_epsilon = layer_norm_epsilon60        self.initializer_range = initializer_range61 62        super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)63 64