xdecoder/Instruct-X-Decoder
163
1U
2 E$�c�. � @ s� d Z ddlZddlmZmZ ddlZddlm mZ ddlm3Z4mZ G dd� dej �Z G dd� dej �Z
G d d5� d6ej �ZG d d � d ej �ZG d
d� dej �Zdd� Zdd� ZdS )z�7Transformer class.8 9Copy-paste from torch.nn.Transformer with modifications:10 * positional encodings are passed in MHattention11 * extra LN at the end of encoder is removed12 * decoder returns a stack of activations from all decoding layers13� N)�List�Optional)�Tensor�nnc s. e Zd Zd� fdd � Zd14d � Zd d
� Z� ZS )� Transformer� � � � 皙�����?�reluFc15 s� t � �� t||||||�}16|r*t�|�nd } t|17|| �| _t||||||�} t�|�}
t| ||
| d�| _ | �18� || _ || _ d S )N)�return_intermediate)
�super�__init__�TransformerEncoderLayerr � LayerNorm�TransformerEncoder�encoder�TransformerDecoderLayer�TransformerDecoder�decoder�_reset_parameters�d_model�nhead)�selfr r �num_encoder_layers�num_decoder_layers�dim_feedforward�dropout�19activation�normalize_beforeZreturn_intermediate_dec�
encoder_layer� encoder_norm�
decoder_layer� decoder_norm�� __class__� �9/data/arXiv/demo/Demo/xdecoder/body/transformer_blocks.pyr s: 20 � �21�zTransformer.__init__c C s* | � � D ] }|�� dkrtj�|� qd S )N� )�22parameters�dimr �init�xavier_uniform_)r �pr' r' r( r 8 s z Transformer._reset_parametersc C s� |j \}}}}|�d��ddd�}|�d��ddd�}|�d��d|d�}|d k r\|�d�}t�|�} | j|||d�}23| j| |24|||d�} | � dd�|25�ddd��26||||�fS )N� r r) )�src_key_padding_mask�pos)�memory_key_padding_maskr1 � query_pos) �shape�flatten�permute� unsqueeze�repeat�torch�27zeros_liker r � transpose�view) r �src�maskZ query_embed� pos_embed�bs�c�h�w�tgt�memory�hsr' r' r( �forward= s 2829 �zTransformer.forward) r r r r r30 r r FF)�__name__�31__module__� __qualname__r r rG �
__classcell__r' r' r% r( r s �$r c s> e Zd Zd� fdd� Zdee ee ee d�dd�Z� ZS ) r Nc s& t � �� t||�| _|| _|| _d S �N)r r � _get_clones�layers�32num_layers�norm)r r! rO rP r% r' r( r O s 33 zTransformerEncoder.__init__)r> r0 r1 c C s8 |}| j D ]}|||||d�}q34| jd k r4| �|�}|S )N��src_maskr0 r1 )rN rP )r r= r> r0 r1 �output�layerr' r' r( rG U s 35 �3637zTransformerEncoder.forward)N)NNN�rH rI rJ r r r rG rK r' r' r% r( r N s ��r c sP e Zd Zd� fdd� Zd ee ee ee ee ee ee d�dd�Z� ZS )38r NFc s, t � �� t||�| _|| _|| _|| _d S rL )r r rM rN rO rP r
)r r# rO rP r
r% r' r( r j s39 40 zTransformerDecoder.__init__��tgt_mask� memory_mask�tgt_key_padding_maskr2 r1 r3 c C s� |} g }41| j D ]2} | | |||||||d�} | jr|42�| �| �� q| jd k rn| �| �} | jrn|43�� |44�| � | jr~t�|45�S | �d�S )NrV r )rN r
�appendrP �popr9 �stackr7 ) r rD rE rW rX rY r2 r1 r3 rS � intermediaterT r' r' r( rG q s. 46�4748495051zTransformerDecoder.forward)NF)NNNNNNrU r' r' r% r( r i s ��r c s� e Zd Zd� fdd� Zee d�dd �Zdee ee ee d �d d
�Zdee ee ee d �dd�Zdee ee ee d �dd�Z � Z52S )r r53 r r Fc s� t � �� tj|||d�| _t�||�| _t�|�| _t�||�| _ t�54|�| _ t�55|�| _ t�|�| _
t�|�| _t|�| _|| _d S �N)r )r r r �MultiheadAttention� self_attn�Linear�linear1�Dropoutr �linear2r �norm1�norm2�dropout1�dropout2�_get_activation_fnr r �r r r r r r r r% r' r( r � s 56 57z TransformerEncoderLayer.__init__�r1 c C s |d kr |S || S rL r' �r �tensorr1 r' r' r( �with_pos_embed� s z&TransformerEncoderLayer.with_pos_embedNrQ c C sx | � ||� }}| j|||||d�d }|| �|� }| �|�}| �| �| �| �|����}|| �|� }| � |�}|S �N��value� attn_mask�key_padding_maskr )58rn r` rg re rd r r rb rh rf )r r= rR r0 r1 �q�k�src2r' r' r( � forward_post� s ��59 60z$TransformerEncoderLayer.forward_postc C sx | � |�}| �||� }}| j|||||d�d }|| �|� }| �|�}| �| �| �| �|����}|| � |� }|S ro )61re rn r` rg rf rd r r rb rh )r r= rR r0 r1 rv rt ru r' r' r( � forward_pre� s 62 ��63 z#TransformerEncoderLayer.forward_prec C s& | j r| �||||�S | �||||�S rL �r rx rw )r r= rR r0 r1 r' r' r( rG � s z TransformerEncoderLayer.forward)r64 r r F)NNN)NNN)NNN� rH rI rJ r r r rn rw rx rG rK r' r' r% r( r � s< � �� �� ��r c s� e Zd Zd� fdd� Zee d�dd �Zdee ee ee ee ee ee d �d d
�Zdee ee ee ee ee ee d �dd�Zdee ee ee ee ee ee d �dd�Z � Z65S )r r66 r r Fc s� t � �� tj|||d�| _tj|||d�| _t�||�| _t�|�| _ t�||�| _67t� |�| _ t� |�| _
t� |�| _t�|�| _t�|�| _t�|�| _t|�| _|| _d S r^ )r r r r_ r` �multihead_attnra rb rc r rd r re rf �norm3rg rh �dropout3ri r r rj r% r' r( r � s 68 69z TransformerDecoderLayer.__init__rk c C s |d kr |S || S rL r' rl r' r' r( rn s z&TransformerDecoderLayer.with_pos_embedNrV c C s� | � ||� } }70| j| |71|||d�d } || �| � }| �|�}| j| � ||�| � ||�|||d�d } || �| � }| �|�}| �| �| � | �72|����} || � | � }| � |�}|S �Nrp r )�query�keyrq rr rs )
rn r` rg re r{ rh rf rd r r rb r} r| ) r rD rE rW rX rY r2 r1 r3 rt ru �tgt2r' r' r( rw s6 ��737475��76 77z$TransformerDecoderLayer.forward_postc C s� | � |�} | �| |� }78} | j|79| | ||d�d } || �| � }| �|�} | j| �| |�| �||�|||d�d } || �| � }| �|�} | �| � | �80| � | ����} || � | � }|S r~ )
re rn r` rg rf r{ rh r| rd r r rb r} ) r rD rE rW rX rY r2 r1 r3 r� rt ru r' r' r( rx % s6 81 ��828384��85 z#TransformerDecoderLayer.forward_prec 86 C s6 | j r | �||||||||�S | �||||||||�S rL ry ) r rD rE rW rX rY r2 r1 r3 r' r' r( rG D s* �87�z TransformerDecoderLayer.forward)r88 r r F)NNNNNN)NNNNNN)NNNNNNrz r' r' r% r( r � s` � �� # �� # ��r c s t �� fdd�t|�D ��S )Nc s g | ]}t �� ��qS r' )�copy�deepcopy)�.0�i��moduler' r( �89<listcomp>g s z _get_clones.<locals>.<listcomp>)r �90ModuleList�range)r� �Nr' r� r( rM f s rM c C s> | dkrt jS | dkr t jS | dkr*t jS td| � d���dS )z,Return an activation function given a stringr �gelu�gluz$activation should be relu/gelu, not �.N)�Fr r� r� � RuntimeError)r r' r' r( ri j s ri )�__doc__r� �typingr r r9 Ztorch.nn.functionalr �91functionalr� r �Moduler r r r r rM ri r' r' r' r( �<module> s ;1M