sumanthd/IndicTrans-MultilingualTranslation
6
1B
2 ��,c�- � @ s� d Z ddlZddlmZ ddlZddlmZmZmZm Z ddl3m Z ddl m
Z
mZ ddlmZ ddlZedd �Zed4d �Zdd
d�ZG dd� d�ZdS )zC5Translate raw text with a trained model. Batches data on-the-fly.6� N)�7namedtuple)�checkpoint_utils�options�tasks�utils)� convert_namespace_to_omegaconf)�pack_constraints�unpack_constraints)� get_symbols_to_strip_from_output�Batchz&ids src_tokens src_lengths constraints� Translationz#src_str hypos pos_scores alignmentsFc # s& � fdd��|r�dd� | D �}x2t | �D ]&\}}d|kr(|�d�^| |< ||< q(W x,t |�D ] \}} ��fdd�| D �||< q\W |r�t|�}8nd }9��| � �\} } �j�j| | |10d�|jj|jj||jj d�j11d d12�}
xJ|
D ]B}|d }|d d
}|d d }|� dd �}t ||||d�V q�W d S )Nc s � | �S )N� )�x)� encode_fnr
�7D:\ai4bharat\indicTrans\inference\custom_interactive.py�encode_fn_target s z&make_batches.<locals>.encode_fn_targetc S s g | ]13}t � �qS r
)�list)�.0�_r
r
r �14<listcomp>% s z make_batches.<locals>.<listcomp>� c s" g | ]}�j j� |�d d d��qS )F)Z15append_eosZadd_if_not_exist)�target_dictionaryZ encode_line)r �16constraint)r �taskr
r r - s )� constraints)�dataset�17max_tokensZ
max_sentences�
max_positionsZignore_invalid_inputsF)�shuffle�id� net_input�18src_tokens� src_lengthsr )�idsr! r" r )
� enumerate�splitr Z"get_interactive_tokens_and_lengthsZget_batch_iteratorZbuild_dataset_for_inferencer r �19batch_size�#skip_invalid_size_inputs_valid_testZnext_epoch_itr�getr )�lines�cfgr r r Zconstrainted_decodingZbatch_constraints�i�lineZconstraint_listZconstraints_tensor�tokens�lengths�itr�batchr# r! r" r r
)r r r r � make_batches s> 2021 22 r1 c @ s0 e Zd Zd dd�Zdd� Zdd� Zd
d23d �Zd S )�24Translator� Fc C s� || _ tjdd�| _| j r8| jj|ddd||d d� n| jj|dd||d d� tj| j|gd �}d25|_d |_d |_d
|_ t26|�| _ t �
| j j� | j jjdk r�d| j j_| j jjd kr�| j jjd kr�d| j j_| j jjr�| j jj| j jjks�td��| j jj�r"| j jj| j jjk�s"td��tj�� �o8| j jj | _ t � | j j �| _ t!�"| j j#j$�}t%j&t �'| j j#j(�|| j | j j)j*| j j)j+dk| j j)j+d�\| _,| _-| j j.| _/| j j0| _1xZ| j,D ]P}|d k�r̐q�| j jj2�r�|�3� | j �r�| j j4j5�s�|�� |�6| j � �q�W | j �7| j,| j j�| _8| j �9| j j:�| _:| j �;| j j<�| _<t �=| j jj>�| _?t j@| j �A� fdd� | j,D ��� | _Ad S )NT)� interactive� subword_nmt������ordered� )�path�27remove_bpe� num_workersr r& � buffer_size)r9 r: r; r& r< )�28input_argsZSRCZTGTFZ
model_configsz1--sampling requires --nbest to be equal to --beamz0--batch-size cannot be larger than --buffer-size)Z
arg_overridesr �suffix�strictZ29num_shardsc S s g | ] }|� � �qS r
)r )r �modelr
r
r r � s z'Translator.__init__.<locals>.<listcomp>)B�constrained_decodingr Zget_generation_parser�parser� set_defaultsZparse_args_and_archZ source_lang� target_langr' Zuser_dirr r* r Zimport_user_module�commonr4 r< r r r& �30generationZsampling�nbestZbeam�AssertionError�torch�cuda� is_available�cpu�use_cudar Z31setup_taskr �ast� literal_eval� common_evalZmodel_overridesr Zload_model_ensemble� split_pathsr9 Z32checkpointZcheckpoint_suffixZcheckpoint_shard_count�modelsZ _model_argsZsource_dictionary�src_dictr �tgt_dictZfp16�halfZdistributed_trainingZpipeline_model_parallelZprepare_for_inference_Zbuild_generator� generatorZbuild_tokenizer� tokenizerZ build_bpe�bpeZload_align_dictZ replace_unk�33align_dictZresolve_max_positionsr )�self�data_dirZcheckpoint_pathr& rA �args� overridesr@ r
r
r �__init__T sx 3435 3637 3839 40 zTranslator.__init__c C s0 | j d k r| j �|�}| jd k r,| j�|�}|S )N)rW �encoderX )rZ r r
r
r r � s41 42 43 zTranslator.encode_fnc C s0 | j d k r| j �|�}| jd k r,| j�|�}|S )N)rX �decoderW )rZ r r
r
r � decode_fn� s44 45 46 zTranslator.decode_fnNc C sP | j r|d krtd��| j s,|d k r,td��|rjd}g }x*t||�D ] \}}|�|d|� � � qDW |}nd}d}g }g } �xt|| j| j| j| j|�D ]�}47|48j �49d�} |50j } |51j }
|52j }| j
r�| �� } |
�� }
|d k r�|�� }d| |
d�i}| jj| j| j||d �}d53d � t| �D �}|�r,d d � |D �}x^tt|54j�� |��D ]F\}\}}t�| | | j�� �}|| }|�|| ||d
|if� �qBW q�W x�t|dd� d�D ]�\}} }}d}| jd k �r�| j� | | jj j �}xr|d t t |�| jj!j"�� D ]R}tj#|d �$� �%� ||d | j&| jdt'| j�d�\}}}| �(|�} | �| � �q�W �q�W | S )Nz5Constraints cant be None in constrained decoding modez1Cannot pass constraints during normal translationTr Fr r )r! r" )r c S s g | ]}g �qS r
r
)r r r
r
r r s z(Translator.translate.<locals>.<listcomp>c S s g | ] }t |��qS r
)r )r �cr
r
r r s r c S s | d S )Nr r
)r r
r
r �<lambda> � z&Translator.translate.<locals>.<lambda>)�key� r- � alignmentr5 )� hypo_tokens�src_strrg rY rT r: Zextra_symbols_to_ignore))rA �55ValueError�zip�appendr1 r* r r r r! �sizer" r rM rJ Zinference_steprV rR �ranger$ r# �tolistr Z strip_padrT �pad�sortedrS �stringrP Z post_process�min�lenrF rG Zpost_process_prediction�intrL rY r56 ra ) rZ �inputsr rA Zmodified_inputs�_inputr Zstart_id�resultsZfinal_translationsr0 Zbszr! r" �sample� translationsZlist_constraintsr+ r ZhyposZ src_tokens_i�id_r ri Zhyporh Zhypo_strrg Zdetok_hypo_strr
r
r � translate� s� 57 58" "59zTranslator.translate)r3 F)N)�__name__�60__module__� __qualname__r^ r ra r| r
r
r
r r2 S s 61qr2 )F)�__doc__rN � collectionsr rI Zfairseqr r r r Zfairseq.dataclass.utilsr Z$fairseq.token_generation_constraintsr r Zfairseq_cli.generater62 �codecsr r r1 r2 r
r
r
r �<module>63 s 6465666