xdecoder/Instruct-X-Decoder
163
1U
2 E$�c� � @ s� d dl Z d dl mZ d dlmZ d dlmZ ddlmZ ddl m3Z4 dd l m Z m
Z
d d5lmZmZ G d d � d ej�Zed
d� �ZdS )� N)�nn)�6functional)�
trunc_normal_� )�register_model� )� configurable)�build_tokenizer�build_lang_encoder)�prompt_engineering�get_prompt_templatesc s^ e Zd Ze� fdd��Zedd� �Zddd �Zdd7d �Zdd d
�Z ddd�Z8ddd�Z � Z S )�LanguageEncoderc s> t � �� || _|| _|| _|| _|| _t�t �9g ��| _ d S �N) �super�__init__� tokenizer�tokenizer_type� lang_encoder� lang_proj�
max_token_numr � Parameter�torch�ones� logit_scale)�selfr r r �lang_projectionr �� __class__� �5/data/arXiv/demo/Demo/xdecoder/language/vlpencoder.pyr s 10zLanguageEncoder.__init__c C s� t |d d �}|d d d }t|d d ||d �}|d d d }|d d d }|d d }t�t�||��}t|dd � |||||d11�S ) N�MODEL�TEXTZ TOKENIZER�VERBOSE�CONTEXT_LENGTHZWIDTH�DIM_PROJg{�G�z�?)�std)r r r r r )r r12 r r r �emptyr ) �cls�cfgr r r r Zdim_langZdim_projectionr r r r � from_config! s �zLanguageEncoder.from_config�defaultFTc s� |s�|r2� fdd�t t� ��D �}|r6|�d� n� }g }g } |D ]N}13�j|14dd�jdd�} | d �� | d �� |�| d � | �| d � qBt�|�} t�| �}
�j| � � |
� � f�d15�}t16�d � |�|� n�t� � �� ��fd d
�}t
� }g }|�r*� D ]&��fdd�|D �}|�||�� �q n|�|� �� |�rRdg}|�||�� tj|dd�}t17�d � |�|� W 5 Q R X d S )Nc s6 g | ].}t � | �d d��dd��dd�ddd��qS )�-other� �-merged�-stuffi' �.)�topk�suffix)r �replace)�.0�label)� class_namesr r �18<listcomp>9 s �z7LanguageEncoder.get_text_embeddings.<locals>.<listcomp>zA background in coco.�19max_lengthT�pt��paddingZ20truncationr7 Zreturn_tensors� input_ids�attention_mask��norm�{}_text_embeddingsc sT �j | dd�jdd�}�j|d �� |d �� f� d�}|jdd �}||�� }|S )21Nr7 Tr8 r9 r; r< r= r ��dim)r r �forward_language�cuda�meanr> )�txts�tokensZclss_embedding)r> r r r �extract_mean_embU s �" z=LanguageEncoder.get_text_embeddings.<locals>.extract_mean_embc s. g | ]&}|� � �d d��dd��dd���qS )r+ r, r- r. )�formatr2 )r3 �template)�clssr r r6 b s r r@ )�range�len�appendr r �squeeze_r �stackrB rC �setattrrH �no_gradr )r r5 �name�is_evalZadd_bgd�promptr> Zarbitary_conceptsr; Zattention_masks�txtrF Zarbitary_tokensZarbitary_attention_masksZtext_embrG � templatesZclss_embeddingsrE r )r5 rJ r> r r �get_text_embeddings5 sP 2223� � 242526 z#LanguageEncoder.get_text_embeddingsc C sr |s.| j |dd| jdd�}dd� |�� D �}n|}| j|d |d f|d �\}}|||d27�}t| d �|�|� |S ) Nr7 Tr8 r9 c S s i | ]\}}||� � �qS r )rC )r3 �key�valuer r r �28<dictcomp>s s z=LanguageEncoder.get_text_token_embeddings.<locals>.<dictcomp>r; r< r= )rF � token_emb� class_embz{}_token_embeddings)r r �items�forward_language_tokenrP rH ) r rE rR �tokenr> rF r[ r\ �retr r r �get_text_token_embeddingsn s" � �z)LanguageEncoder.get_text_token_embeddingsc C sz | j |� }|d }| jdkrB|t�|�d��|d jdd�f }n|d d �df }|| j }|rv||jddd�d }|S � NZlast_hidden_state�clipr �����r@ T�rA �keepdim�H�����z>�r r r �arange�size�argmaxr r> )r �textsr> �xr r r rB } s 2930&31z LanguageEncoder.forward_languagec C s� | j |� }|d }| jdkrB|t�|�d��|d jdd�f }n|d d �df }|| j }|| j }|r�||jddd�d }||jddd�d }||fS rb rh )r rl r> rm Ztoken_xZclass_xr r r r^ � s 3233&3435z&LanguageEncoder.forward_language_tokenc C sR |rd S ||j ddd�d }t| d�|��}| j�� | |�d��dd� }|S ) Nrd Tre rg r? r r r )r> �getattrrH r �exp� unsqueeze� transpose)r �v_embrR �fake�t_emb�outputr r r �compute_similarity� s z"LanguageEncoder.compute_similarity)r* FFTT)r* FF)T)F)r* F)
�__name__�36__module__� __qualname__r r � classmethodr) rW ra rB r^ rv �
__classcell__r r r r r
s 37389394041r
c K s t | �S r )r
)r( �kwargsr r r �get_language_model� s r} )r r �torch.nnr �F�timm.models.layersr �registryr �utilsr Z LangEncoderr r42 �43utils.miscr r �Moduler
r} r r r r �<module> s