Team Ai
Apppublic

David310/Detect_AI-generated_Image

sourceHugging Faceupdated 2y agoView on Hugging Face
4likes
rootmodels__pycache__vision_transformer.cpython-38.pyc
vision_transformer.cpython-38.pyc124 linesDownload Raw Back to __pycache__
1U

2��^fD�@s�ddlZddlmZddlmZddlmZmZmZm	Z	m3Z4ddlZddlm
Z
ddlmZddlmZzddlmZWn ek5r�dd	lmZYnXd6ddd
d�ZGdd�de	�ZGdd�de
j�ZGdd�de
j�ZGdd�de
j�ZGdd�de
j�Zee e e e e e!e!eed�7dd�Z"d,e!e!eed�dd �Z#d-e!e!eed�d!d"�Z$d.e!e!eed�d#d$�Z%d/e!e!eed�d%d&�Z&d0e e d(ee!d(d)�d*d+�Z'dS)1�N)�OrderedDict)�partial)�Any�Callable�List�8NamedTuple�Optional�)�ConvNormActivation)�_log_api_usage_once)�load_state_dict_from_url)�load_urlz9https://download.pytorch.org/models/vit_b_16-c867db91.pthz9https://download.pytorch.org/models/vit_b_32-d86f8d99.pthz9https://download.pytorch.org/models/vit_l_16-852ce7e3.pthz9https://download.pytorch.org/models/vit_l_32-c7638314.pth)�vit_b_16�vit_b_32�vit_l_16�vit_l_32c@sVeZdZUeed<eed<eed<ejZedej	fed<ej9Zedej	fed<dS)�ConvStemConfig�out_channels�kernel_size�stride.�10norm_layer�activation_layerN)�__name__�11__module__�__qualname__�int�__annotations__�nn�BatchNorm2drr�Module�ReLUr�r!r!�A/root/autodl-tmp/UniversalFakeDetect/models/vision_transformer.pyr"s1213rcs*eZdZdZeeed��fdd�Z�ZS)�MLPBlockzTransformer MLP block.)�in_dim�mlp_dim�dropoutcs�t���t�||�|_t��|_t�|�|_t�||�|_	t�|�|_14tj�|jj
�tj�|j	j
�tjj|jjdd�tjj|j	jdd�dS)N�����ư>��std)�super�__init__r�Linear�linear_1�GELU�act�Dropout�	dropout_1�linear_2�	dropout_2�init�xavier_uniform_�weight�normal_�bias)�selfr$r%r&��	__class__r!r"r+-s1516zMLPBlock.__init__)rrr�__doc__r�floatr+�
__classcell__r!r!r:r"r#*sr#c	sZeZdZdZeejdd�feeeeee	de17jjfd��fdd�
Ze18j
d�d	d19�Z�ZS)�EncoderBlockzTransformer encoder block.r'��eps.)�	num_heads�20hidden_dimr%r&�attention_dropoutrcsVt���||_||�|_tj|||dd�|_t�|�|_||�|_	t21|||�|_dS)NT)r&�batch_first)r*r+rB�ln_1r�MultiheadAttention�self_attentionr0r&�ln_2r#�mlp)r9rBrCr%r&rDrr:r!r"r+>s	222324zEncoderBlock.__init__��inputcCsjt�|��dkd|j���|�|�}|j|||dd�\}}|�|�}||}|�|�}|�|�}||S)N�z2Expected (seq_length, batch_size, hidden_dim) got F)�query�key�value�need_weights)	�torch�_assert�dim�shaperFrHr&rIrJ)r9rL�x�_�yr!r!r"�forwardSs25262728zEncoderBlock.forward�rrrr<rr�	LayerNormrr=rrRrr+�TensorrYr>r!r!r:r"r?;s	��r?cs^eZdZdZeejdd�feeeeeeee	de29jjfd��fdd�
Ze30j
d�d	d31�Z�ZS)�Encoderz?Transformer Model Encoder for sequence to sequence translation.r'r@.)�32seq_length�33num_layersrBrCr%r&rDrc	s~t���t�t�d||�jdd��|_t�|�|_	t34�}	t|�D] }35t||||||�|	d|36��<qBt�
|	�|_||�|_dS)Nr	g{�G�z�?r(�encoder_layer_)r*r+r�	ParameterrR�emptyr7�
pos_embeddingr0r&r�ranger?�37Sequential�layers�ln)r9r^r_rBrCr%r&rDrrf�ir:r!r"r+bs38�zEncoder.__init__rKcCs<t�|��dkd|j���||j}|�|�|�|���S)NrMz2Expected (batch_size, seq_length, hidden_dim) got )rRrSrTrUrcrgrfr&)r9rLr!r!r"rYs39zEncoder.forwardrZr!r!r:r"r]_s��r]cs�eZdZdZddddeejdd�dfeeeeeeeeee	ee40dejjfe	e
ed��fd	d41�
Zejejd�dd
�Zejd�dd�Z�ZS)�VisionTransformerz;Vision Transformer as per https://arxiv.org/abs/2010.11929.�i�Nr'r@.)�42image_size�43patch_sizer_rBrCr%r&rD�num_classes�representation_sizer�conv_stem_configsc
s�t���t|�t�||dkd�||_||_||_||_||_	||_44|	|_|45|_||_
|dk	r�t��}
d}t|�D]:\}}|
�d|��t||j|j|j|j
|jd��|j}qx|
�dtj||dd��|
|_ntjd|||d	�|_||d46}t�t�dd|��|_|d7}t||||||||�|_||_t�}|47dk�rRt� ||	�|d<n,t� ||48�|d<t�!�|d
<t� |49|	�|d<t�|�|_"t#|jtj��r�|jj$|jjd|jjd}tj%j&|jj't(�)d|�d�|jj*dk	�rbtj%�+|jj*�nj|jj,dk	�rbt#|jj,tj��rbtj%j-|jj,j'dt(�)d|jj,j�d�|jj,j*dk	�rbtj%�+|jj,j*�t.|j"d��r�t#|j"j/tj ��r�|j"j/j0}tj%j&|j"j/j't(�)d|�d�tj%�+|j"j/j*�t#|j"j1tj ��r�tj%�+|j"j1j'�tj%�+|j"j1j*�dS)Nrz&Input shape indivisible by patch size!rM�
conv_bn_relu_)�in_channelsrrrrr�	conv_lastr	)rqrr)rqrrr��head�50pre_logitsr/r(rjg@)�meanr))2r*r+rrRrSrkrlrCr%rDr&rmrnrrre�	enumerate�51add_moduler52rrrr�Conv2d�	conv_projra�zeros�class_tokenr]�encoderr^rr,�Tanh�heads�53isinstancerqr4�
trunc_normal_r6�math�sqrtr8�zeros_rrr7�hasattrru�in_featuresrt)r9rkrlr_rBrCr%r&rDrmrnrro�seq_proj�
prev_channelsrh�conv_stem_layer_configr^�heads_layers�fan_inr:r!r"r+�s�54�����5556  � 57 zVisionTransformer.__init__)rV�returnc	Csx|j\}}}}|j}t�||jkd�t�||jkd�||}||}|�|�}|�||j||�}|�ddd�}|S)NzWrong image height!zWrong image width!rrsr	)	rUrlrRrSrkrz�reshaperC�permute)	r9rV�n�c�h�w�p�n_h�n_wr!r!r"�_process_input�s58z VisionTransformer._process_input)rVcCs�i}|�|�}|jd}|j�|dd�}tj||gdd�}|�|�}|dd�dd�f}t|j|j	�}}|�59||||j��dddd�|d<|dd�df}||d<|�
|�}||d	<|S)60Nr�����r	�rTrMrs�f4�penultimate�logits)r�rUr|�expandrR�catr}rrkrl�viewrCr�r)r9rV�outr��batch_class_tokenZimg_feature�H�Wr!r!r"rYs616263"64zVisionTransformer.forward)rrrr<rrr[rr=rrrRrrrr+r\r�rYr>r!r!r:r"ri�s.65�66�iri)67�archrlr_rBrCr%�68pretrained�progress�kwargsr�c	Ksb|�dd�}	tf|	|||||d�|��}69|r^|tkrDtd|�d���tt||d�}|70�|�|71S)Nrk��)rkrlr_rBrCr%z+No checkpoint is available for model type 'z'!)r�)�popri�72model_urls�73ValueErrorr�load_state_dict)r�rlr_rBrCr%r�r�r�rk�model�74state_dictr!r!r"�_vision_transformer!s"��7576r�FT)r�r�r�r�cKs tfdddddd||d�|��S)aT77    Constructs a vit_b_16 architecture from78    `"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale" <https://arxiv.org/abs/2010.11929>`_.79 80    Args:81        pretrained (bool): If True, returns a model pre-trained on ImageNet82        progress (bool): If True, displays a progress bar of the download to stderr83    r�����r�rlr_rBrCr%r�r��r��r�r�r�r!r!r"rAs	�	�rcKs tfdddddd||d�|��S)aT84    Constructs a vit_b_32 architecture from85    `"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale" <https://arxiv.org/abs/2010.11929>`_.86 87    Args:88        pretrained (bool): If True, returns a model pre-trained on ImageNet89        progress (bool): If True, displays a progress bar of the download to stderr90    r� r�r�r�r�r�r�r!r!r"rWs	�	�rcKs tfdddddd||d�|��S)aT91    Constructs a vit_l_16 architecture from92    `"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale" <https://arxiv.org/abs/2010.11929>`_.93 94    Args:95        pretrained (bool): If True, returns a model pre-trained on ImageNet96        progress (bool): If True, displays a progress bar of the download to stderr97    rr����r�r�r�r!r!r"rms	�	�rcKs tfdddddd||d�|��S)aT98    Constructs a vit_l_32 architecture from99    `"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale" <https://arxiv.org/abs/2010.11929>`_.100 101    Args:102        pretrained (bool): If True, returns a model pre-trained on ImageNet103        progress (bool): If True, displays a progress bar of the download to stderr104    rr�r�r�r�r�r�r�r�r!r!r"r�s	�	�r�bicubiczOrderedDict[str, torch.Tensor])rkrl�model_state�interpolation_mode�reset_headsr�cCsX|d}|j\}}}|dkr,td|j����||dd}	|	|k�rT|d8}|	d8}	|dd�dd�dd�f}105|dd�dd�dd�f}|�ddd�}tt�|��}t�|||kd�|�d|||�}||}
t	j106j||
|dd	�}|�d||	�}|�ddd�}tj|107|gdd108�}||d<|�rTt
�}|��D]\}}|�d��s0|||<�q0|}|S)a�This function helps interpolating positional embeddings during checkpoint loading,109    especially when you want to apply a pre-trained model on images with different resolution.110 111    Args:112        image_size (int): Image size of the new model.113        patch_size (int): Patch size of the new model.114        model_state (OrderedDict[str, torch.Tensor]): State dict of the pre-trained model.115        interpolation_mode (str): The algorithm used for upsampling. Default: bicubic.116        reset_heads (bool): If true, not copying the state of heads. Default: False.117 118    Returns:119        OrderedDict[str, torch.Tensor]: A state dict which can be loaded into the new model.120    zencoder.pos_embeddingr	z%Unexpected position embedding shape: rsNrz#seq_length is not a perfect square!T)�size�mode�
align_cornersr�r)rUr�r�rr�r�rRrSr�r�121functional�interpolater�r�items�122startswith)rkrlr�r�r�rcr�r^rC�new_seq_length�pos_embedding_token�pos_embedding_img�
seq_length_1d�new_seq_length_1d�new_pos_embedding_img�new_pos_embedding�model_state_copy�k�vr!r!r"�interpolate_embeddings�s@123�r�)FT)FT)FT)FT)r�F)(r��collectionsr�	functoolsr�typingrrrrrrR�torch.nnrZvision_transformer_miscr124Zvision_transformer_utilsrZ	torch.hubr�ImportError�torch.utils.model_zoor
r�rrer#rr?r]ri�strr�boolr�rrrrr�r!r!r!r"�<module>s^�$&� ��