Team Ai
Apppublic

xdecoder/Instruct-X-Decoder

sourceHugging Faceafl-3.0updated 3y agoView on Hugging Face
163likes
xdecoder.cpython-38.pyc141 linesDownload Raw Back to __pycache__
1U

2E$�cS}�@s�ddlZddlmZddlZddlmZmZddlmZddl	m3Z4ddlmZddl
mmZddlmZd	d5lmZd	dlmZGdd
�d
ej�ZGdd�dej�ZGdd�dej�Zdd�ZGdd�dej�ZGdd�dej�Zedd��ZdS)�N)�Optional)�nn�Tensor)�6functional)�
trunc_normal_)�Conv2d�)�register_decoder�)�configurable)�PositionEmbeddingSinecs�eZdZd�fdd�	Zdd�Zeed�d	d7�Zdeeeeeed�d
d�Zdeeeeeed�dd�Z	deeeeeed�dd�Z8�ZS)�SelfAttentionLayer��reluFcsPt���tj|||d�|_t�|�|_t�|�|_t	|�|_9||_|��dS�N)�dropout)
�super�__init__r�MultiheadAttention�	self_attn�	LayerNorm�norm�Dropoutr�_get_activation_fn�10activation�normalize_before�_reset_parameters��self�d_model�nheadrrr��	__class__��7/data/arXiv/demo/Demo/xdecoder/body/decoder/xdecoder.pyrs1112zSelfAttentionLayer.__init__cCs*|��D]}|��dkrtj�|�qdS�Nr��13parameters�dimr�init�xavier_uniform_�r�pr#r#r$r+sz$SelfAttentionLayer._reset_parameters��poscCs|dkr|S||S�Nr#�r�tensorr.r#r#r$�with_pos_embed0sz!SelfAttentionLayer.with_pos_embedN��tgt_mask�tgt_key_padding_mask�	query_poscCsD|�||�}}|j|||||d�d}||�|�}|�|�}|S�N)�value�	attn_mask�key_padding_maskr)r2rrr)r�tgtr4r5r6�q�k�tgt2r#r#r$�forward_post3s��14zSelfAttentionLayer.forward_postcCsD|�|�}|�||�}}|j|||||d�d}||�|�}|Sr7)rr2rr)rr;r4r5r6r>r<r=r#r#r$�forward_pre?s15��zSelfAttentionLayer.forward_precCs&|jr|�||||�S|�||||�Sr/�rr@r?)rr;r4r5r6r#r#r$�forwardKs��zSelfAttentionLayer.forward)rrF)NNN)NNN)NNN��__name__�16__module__�__qualname__rrrrr2r?r@rB�
__classcell__r#r#r!r$r
s<�
��
��
��r
cs�eZdZd�fdd�	Zdd�Zeed�d	d17�Zdeeeeeeeed�d
d�Zdeeeeeeeed�dd�Z	deeeeeeeed�dd�Z18�ZS)�CrossAttentionLayerrrFcsPt���tj|||d�|_t�|�|_t�|�|_t	|�|_19||_|��dSr)
rrrr�multihead_attnrrrrrrrrrr!r#r$rXs2021zCrossAttentionLayer.__init__cCs*|��D]}|��dkrtj�|�qdSr%r&r+r#r#r$resz%CrossAttentionLayer._reset_parametersr-cCs|dkr|S||Sr/r#r0r#r#r$r2jsz"CrossAttentionLayer.with_pos_embedN��memory_mask�memory_key_padding_maskr.r6c	CsH|j|�||�|�||�|||d�\}}||�|�}|�|�}||fS�N)�query�keyr8r9r:)rIr2rr�	rr;�memoryrKrLr.r6r>�avg_attnr#r#r$r?ms22�2324z CrossAttentionLayer.forward_postc	CsH|�|�}|j|�||�|�||�|||d�\}}||�|�}||fSrM)rrIr2rrPr#r#r$r@zs2526�27zCrossAttentionLayer.forward_precCs.|jr|�||||||�S|�||||||�Sr/rA)rr;rQrKrLr.r6r#r#r$rB�s28�29�zCrossAttentionLayer.forward)rrF)NNNN)NNNN)NNNNrCr#r#r!r$rHVsH�
������rHcsPeZdZd�fdd�	Zdd�Zeed	�d30d�Zdd
�Zdd�Z	dd�Z31�ZS)�FFNLayer�rrFcsZt���t�||�|_t�|�|_t�||�|_t�|�|_	t32|�|_||_|�
�dSr/)rrr�Linear�linear1rr�linear2rrrrrr)rr�dim_feedforwardrrrr!r#r$r�s3334zFFNLayer.__init__cCs*|��D]}|��dkrtj�|�qdSr%r&r+r#r#r$r�szFFNLayer._reset_parametersr-cCs|dkr|S||Sr/r#r0r#r#r$r2�szFFNLayer.with_pos_embedc	Cs8|�|�|�|�|����}||�|�}|�|�}|Sr/)rWrrrVr�rr;r>r#r#r$r?�s35zFFNLayer.forward_postc	Cs8|�|�}|�|�|�|�|����}||�|�}|Sr/)rrWrrrVrYr#r#r$r@�s36zFFNLayer.forward_precCs|jr|�|�S|�|�Sr/rA)rr;r#r#r$rB�s37zFFNLayer.forward)rTrrFrCr#r#r!r$rS�s�rScCs>|dkrtjS|dkrtjS|dkr*tjStd|�d���dS)z,Return an activation function given a stringr�gelu�gluz$activation should be relu/gelu, not �.N)�FrrZr[�RuntimeError)rr#r#r$r�srcs(eZdZdZ�fdd�Zdd�Z�ZS)�MLPz5 Very simple multi-layer perceptron (also called FFN)csJt���||_|g|d}t�dd�t|g|||g�D��|_dS)Nrcss|]\}}t�||�VqdSr/)rrU)�.0�nr=r#r#r$�	<genexpr>�szMLP.__init__.<locals>.<genexpr>)rr�38num_layersr�39ModuleList�zip�layers)r�	input_dim�40hidden_dim�41output_dimrc�hr!r#r$r�s42zMLP.__init__cCs<t|j�D],\}}||jdkr.t�||��n||�}q43|Sr%)�	enumeraterfrcr]r)r�x�i�layerr#r#r$rB�s&zMLP.forward)rDrErF�__doc__rrBrGr#r#r!r$r_�sr_cs�eZdZdZedejeeeeeeeeee	eed�
�fdd�
�Z44edd��Zdddd	ifd45d�Z
dddd	ifdd
�Zddd�Zejjdd��Z�ZS)�"MultiScaleMaskedTransformerDecoder�T)
�lang_encoderrh�dim_proj�num_queries�46contxt_len�nheadsrX�47dec_layers�pre_norm�mask_dim�task_switch�captioning_step�enforce_input_projectc	s�t���|std��||_|d}t|dd�|_||_|48|_||_t	�49�|_t	�50�|_t	�51�|_
t|j�D]L}|j�t||d|d��|j�t||d|d��|j
�t||	d|d��qlt	�|�|_||_t	�||�|_t	�||�|_d|_t	�|j|�|_t	�52�|_t|j�D]N}||k�s&|�rN|j�t||d	d53��t�|jd�n|j�t	����q|
|_ ||_!|j d�r�t"|||d�|_#t	�$t%�&||��|_'t(|j'd
d�|
d�r�t"||dd�|_)|
d�rt	�$t%�&||��|_*t(|j*d
d�t	�||�|_+||_,t%�-d	||||f��.�}d|dd�d|�|d�f<t%j/t%�0d	||f�d	d��.�|dd�|d�|d�f<d|dd�d|d	�|d	|�f<d|dd�|d	|�d|d	�f<|�1d|�dS)a54        NOTE: this interface is experimental.55        Args:56            in_channels: channels of the input features57            mask_classification: whether to add mask classifier or not58            num_classes: number of classes59            hidden_dim: Transformer feature dimension60            num_queries: number of queries61            nheads: number of heads62            dim_feedforward: feature dimension in feedforward network63            enc_layers: number of Transformer encoder layers64            dec_layers: number of Transformer decoder layers65            pre_norm: whether to use pre-LayerNorm or not66            mask_dim: mask feature dimension67            enforce_input_project: add input project 1x1 conv even if input68                channels and hidden dim is identical69        z&Only support mask classification modelrqT)�	normalizer)rr rr)rrXrrr70r)�kernel_size������maskg{�G�z�?)�std�bbox��71captioningN)�diagonal�self_attn_mask)2rr�AssertionError�mask_classificationr�pe_layer�	num_headsrcrurrd�!transformer_self_attention_layers�"transformer_cross_attention_layers�transformer_ffn_layers�range�appendr
rHrSr�decoder_normrt�	Embedding�72query_feat�query_embed�num_feature_levels�level_embed�73input_projr�weight_init�c2_xavier_fill�74Sequentialrzrrr_�75mask_embed�	Parameter�torch�empty�class_embedr�76bbox_embed�caping_embed�pos_embed_capingr{�zeros�bool�triu�ones�register_buffer)rrr�in_channelsr�rhrsrtrurvrXrwrxryrzr{r|�N_steps�_r�r!r#r$r�s�%77787980��	��	��	8182834""z+MultiScaleMaskedTransformerDecoder.__init__c	Cs�i}||d<||d<||d<|dd}|dd}|d|d<|dd	|d84<|d|d<|dd
d|d<|d|d<|d|d<|ddks�t�|dd|d<|d|d<|d|d<|d|d<|d|d<|d�dd �|d!<|S)"Nrrr�r��MODEL�ENCODER�DECODERZ85HIDDEN_DIMrh�DIM_PROJrs�NUM_OBJECT_QUERIESrt�TEXTZCONTEXT_LENGTHru�NHEADSrv�DIM_FEEDFORWARDrXZ86DEC_LAYERSrrw�PRE_NORMrxZENFORCE_INPUT_PROJr|�MASK_DIMryrz�87CAPTIONING�STEP�2r{)r��get)	�cls�cfgr�rrr��extra�ret�enc_cfg�dec_cfgr#r#r$�from_config`s(z.MultiScaleMaskedTransformerDecoder.from_configN�segc 88Cs�|dkr |j|||||||d�St|�|jks2t�g}g}	g}89~t|j�D]�}|90�||jdd��|	�|�||d��d��|�|j	|||��d�|j91j|ddd�df�|	d�ddd�|	d<|d�ddd�|d<qJ|dj\}}
}|j
j�d��d|
d�}|jj�d��d|
d�}g}g}g}g}g}d}|j�r�|dk�r�|jd	�r�tjd92d�|D�dd��dd�}|����}tj||fdd�}||jj�d��d|
d�7}tj||fdd�}|j�|jd|jdd�}�n�|j�r�|d
k�s|dk�r|jd�s4|j�rL|dk�rL|jdd�rL|jdd�d|j�d|j�f�|jd|jdd�}|d}|����}tjd|j|jdt|�|j|jdt|�f|jd����|jd|jdd�}||dd�d|j�d|j�f<d|dd�|jd�|jd�f<|}tj||dd�fdd�}tj||dd�fdd�}n6|jdd�d|j�d|j�f�|jd|jdd�}|j|||93d|d�}|d}|�|d�|�|d�|�|d�|�|d�|�|d�t|j �D�]$}||j}d|t�!|�"d�|jdk�<|j�rl|dk�rl|jd	�rltj|t�#|dd�d|j$�dd�f�fdd�}|j%|||||d|	||d�\}}|j�r�|d
k�s�|dk�r�|jd�s�|j�r|dk�r|jdd�rtj||fdd�}tj||fdd�}|j&|||d|d�}|j'||�}|j�r6|d
k�s@|dk�rL|jd�sn|j�r�|dk�r�|jdd�r�|t|�d�}|dt|��}|dt|��}|j|||94|d|j||d�}|d}|�|d�|�|d�|�|d�|�|d�|�|d��q�t|�|j dk�s0t�|dk�rr|d|ddd�t(|dd�|dd��D�d�}|S|d|d|d|d|�)|j*�r�|nd|||�d �}|SdS)!N�captioning_infer)r��target_queries�95target_vlp�taskr������rqrrr�vlpr�cSsg|]}|d�qS)�caption_tokensr#)r`�captionr#r#r$�96<listcomp>�sz>MultiScaleMaskedTransformerDecoder.forward.<locals>.<listcomp>�r(r��grounding_eval�	grounding�	openimage�grounding_tokens��deviceF��attn_mask_target_sizer�r9�
outputs_class�outputs_mask�outputs_bbox�outputs_caption�outputs_captiontingrJr3�r��layer_idr�cSsg|]\}}||d��qS))�pred_captionings�
pred_captionsr#)r`rl�yr#r#r$r��s)r�r��aux_outputs)�pred_logits�97pred_masks�98pred_boxesr�r�)+�forward_captioning�lenr�r�r�r��shaper��flattenr�r��weight�permuter��	unsqueeze�repeatr��trainingrzr��cat�	transpose�detach�cloner�r�r�rtr�r�r��forward_prediction_headsrc�where�sum�99zeros_likerur�r�r�re�
_set_aux_lossr�) rrl�
mask_featuresr�r�r�r�r��srcr.�	size_listrmr��bsr��outputZpredictions_classZpredictions_maskZpredictions_bboxZpredictions_captionZpredictions_captioning�
self_tgt_mask�caping_lang_embedZ_caping_lang_embedr�Z_grounding_tokensZpad_tgt_mask�resultsr9�level_indexrR�outr#r#r$rB�s�8  (���6X61000�101(�����(���"102"���	z*MultiScaleMaskedTransformerDecoder.forwardc$CsJt|�|jkst�g}g}	g}103~t|j�D]�}|104�||jdd��|	�|�||d��d��|�|j|||��d�|j	j105|ddd�df�|	d�ddd�|	d<|d�ddd�|d<q*|dj\}}
}|jj106�
d��d|
d�}|jj107�
d��d|
d�}|d�|
d�}d}d|k�r^|d|dd�dt|dd��f<t|dd�d}|jj108�
d��d|
d�}|jjjj109}t||j�D�]P}|j�|f�d�dd�}tj||fdd�}||7}tj||fdd�}|j|||110d|d	�}|d111}t|j�D�]�}||j}d|t�|�d�|jdk�<tj|t�|dd�d|j�dd�f�fdd�}|j�|jd|jdd�}|ddk	�r(|j\}
}}|
|jk�s�td
��|d}|� |
||112|dd|113|dd�}t!j"|d�#�|114|ddd��$�d}d|dd�|j%d�|f<|� |
||�}|j&|||||d|	||d�\}}|j'|||d|d�}|j(||�}|j|||115|d|j||d�}|d116}�q|d} | |�)�} | dd�|f�*d�d|dd�|df<�q�|jj+j,|dd�}!g}"|!D]>}|�-d�d}|�.dd�}|�.dd�}|�/�}|"�|��q�||"d�}#|#S)Nr�rqrrr�start_token�tokenr�r�r9F�captioning_maskz/Only support single image referring captioning.r117r/�nearest)�mode)rrTrJr3r�r�)Zskip_special_tokensz
<|endoftext|>�z<|startoftext|>)r��118pred_texts)0r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�r�rrZtoken_embeddingr{Zforward_language_tokenr�r�r�r�rcr�r�r�rur�r��reshaper]�interpolate�floatr�rtr�r�r��t�max�	tokenizerZbatch_decode�split�replace�strip)$rrlr�r�r�r�r�r�r�r.r�rmr�r�Zquery_embed_r�Zcaping_lang_tokenZstart_idr�Z119token_embsZcap_idxr�r�r�r�r9r�r�Znq�whZcap_maskrRZpred_captions_gen�textsZ	texts_newr�r#r#r$r�s�8120$1210((�122��".�z5MultiScaleMaskedTransformerDecoder.forward_captioningrcCs|�|�}|�dd�}|jdrN|dks0|dkrN|dd�|jd�f|j}nd}||jddd�d	}|dd�d|jd�f}	|dd�|jd|j�f}123|124|	�dd125��d�dd�ddd�df}||dd�d|jd�fjddd�}126|j�r|dk�s|dk�r|jd
�s:|j�r�|dk�r�|jdd
�r�t	j127|dd�d|jd�f|128|dd�|jd129|jd�ffdd�}n(t	j130|dd�d|jd�f|131fdd�}||j}|jj
||jd�r�|j�p�|dkd�}
|jd�s�|jdd�r�|�|�}t	�d||�}tj||ddd�}|���d132��d��d|jdd��dd�dk��}|��}|dd�|j|jd�f�d�nDd}t	jt|jdd133��|d|dg|jd��|jdd���}dd�tt|��D�}|jd�r�|� |�}d}|jd�r|}|
|||||d�}|S)Nrrr�r�r�rT)r(�keepdimgH�����z>rqr�r�r�r�r�r�)�fakezbqc,bchw->bqhw�bilinearF)�sizer�
align_cornersg�?r�cSsg|]}d�qSr/r#)r`rmr#r#r$r��szOMultiScaleMaskedTransformerDecoder.forward_prediction_heads.<locals>.<listcomp>r�r�)r�r�r�r9r�r�)!r�r�rzrtr�r�softmaxr�r�r�r�r�rrZcompute_similarityr��einsumr]r�sigmoidr�r�r�r�r�r��fill_r��listr�r�r�r�r�)rr�r�r�r�r�Zdecoder_outputr�Znorm_decoder_outputZ	obj_token�	cls_token�simr�r�r�r�r9r�r�r�r#r#r$r�isX134*((���H(135*1362$@137�z;MultiScaleMaskedTransformerDecoder.forward_prediction_headscCsX|jr>dd�t|dd�|dd�|dd�|dd��D�Sdd�|dd�D�SdS)NcSs"g|]\}}}}||||d��qS))r�r�r�r�r#)r`�a�b�c�dr#r#r$r��s138�zDMultiScaleMaskedTransformerDecoder._set_aux_loss.<locals>.<listcomp>rcSsg|]}d|i�qS)r�r#)r`rr#r#r$r��s)r�re)rr�Zoutputs_seg_masksZ
outputs_boxesZoutputs_captionsr#r#r$r��s139,�z0MultiScaleMaskedTransformerDecoder._set_aux_loss)T)rr�)rDrErF�_versionrr�Module�intr��dictr�classmethodr�rBr�r�r��jit�unusedr�rGr#r#r!r$rp�s4��140#f141CrpcCst|||||�Sr/)rp)r�r�rrr�r�r#r#r$�get_masked_transformer_decoder�sr%)�logging�typingrr�rr�torch.nnrr]�timm.models.layersr�detectron2.layersrZfvcore.nn.weight_initr��registryr	�utilsr�modulesrrr
rHrSrr_rpr%r#r#r#r$�<module>s*:>+d