replit/replit-code-v1_5-3b

1from typing import Dict, List, Optional, Type, Union2import torch3 4def _cast_if_autocast_enabled(tensor: torch.Tensor) -> torch.Tensor:5    if torch.is_autocast_enabled():6        if tensor.device.type == 'cuda':7            dtype = torch.get_autocast_gpu_dtype()8        elif tensor.device.type == 'cpu':9            dtype = torch.get_autocast_cpu_dtype()10        else:11            raise NotImplementedError()12        return tensor.to(dtype=dtype)13    return tensor14 15class LPLayerNorm(torch.nn.LayerNorm):16 17    def __init__(self, normalized_shape: Union[int, List[int], torch.Size], eps: float=1e-05, elementwise_affine: bool=True, device: Optional[torch.device]=None, dtype: Optional[torch.dtype]=None):18        super().__init__(normalized_shape=normalized_shape, eps=eps, elementwise_affine=elementwise_affine, device=device, dtype=dtype)19 20    def forward(self, x: torch.Tensor) -> torch.Tensor:21        module_device = x.device22        downcast_x = _cast_if_autocast_enabled(x)23        downcast_weight = _cast_if_autocast_enabled(self.weight) if self.weight is not None else self.weight24        downcast_bias = _cast_if_autocast_enabled(self.bias) if self.bias is not None else self.bias25        with torch.autocast(enabled=False, device_type=module_device.type):26            return torch.nn.functional.layer_norm(downcast_x, self.normalized_shape, downcast_weight, downcast_bias, self.eps)27 28def rms_norm(x: torch.Tensor, weight: Optional[torch.Tensor]=None, eps: float=1e-05) -> torch.Tensor:29    output = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps)30    if weight is not None:31        return output * weight32    return output33 34class RMSNorm(torch.nn.Module):35 36    def __init__(self, normalized_shape: Union[int, List[int], torch.Size], eps: float=1e-05, weight: bool=True, dtype: Optional[torch.dtype]=None, device: Optional[torch.device]=None):37        super().__init__()38        self.eps = eps39        if weight:40            self.weight = torch.nn.Parameter(torch.ones(normalized_shape, dtype=dtype, device=device))41        else:42            self.register_parameter('weight', None)43 44    def forward(self, x: torch.Tensor) -> torch.Tensor:45        return rms_norm(x.float(), self.weight, self.eps).to(dtype=x.dtype)46 47class LPRMSNorm(RMSNorm):48 49    def __init__(self, normalized_shape: Union[int, List[int], torch.Size], eps: float=1e-05, weight: bool=True, dtype: Optional[torch.dtype]=None, device: Optional[torch.device]=None):50        super().__init__(normalized_shape=normalized_shape, eps=eps, weight=weight, dtype=dtype, device=device)51 52    def forward(self, x: torch.Tensor) -> torch.Tensor:53        downcast_x = _cast_if_autocast_enabled(x)54        downcast_weight = _cast_if_autocast_enabled(self.weight) if self.weight is not None else self.weight55        with torch.autocast(enabled=False, device_type=x.device.type):56            return rms_norm(downcast_x, downcast_weight, self.eps).to(dtype=x.dtype)57NORM_CLASS_REGISTRY: Dict[str, Type[torch.nn.Module]] = {'layernorm': torch.nn.LayerNorm, 'low_precision_layernorm': LPLayerNorm, 'rmsnorm': RMSNorm, 'low_precision_rmsnorm': LPRMSNorm}