Team Ai
Modelpublic

OneScience-Group/DiffDock

sourceHugging Facemitupdated 2mo agoView on Hugging Face
0likes53downloads
rootscripts__pycache__train_diffdock.cpython-310.pyc
train_diffdock.cpython-310.pyc101 linesDownload Raw Back to __pycache__
1o

2�aOj�>�@sjddlZddlZddlZddlZddlZddlmZddlmZddl	m3Z4ddlZddl
Z
ddlZddlmZddlmZddlmZmZmZmZddlmZmZmZdd	lmZzd5dlm Z Wne!yqddlm Z Ynwdd
�Z"dd�Z#dd�Z$dd�Z%dd�Z&dd�Z'dd�Z(dd�Z)dd�Z*dd�Z+d d!�Z,d"d#�Z-d$d%�Z.d&d'�Z/e0d(kr�e/�dSdS))�N)�partial)�Path��SimpleNamespace)�construct_loader)�6t_to_sigma)�inference_epoch_fix�
loss_function�7test_epoch�train_epoch)�ExponentialMovingAverage�get_optimizer_and_scheduler�save_yaml_file)�validate_training_entrypoint�)�build_score_modelcCs t��}|jdddd�|��S)Nz--configTz!Path to the training YAML config.)�required�help)�argparse�ArgumentParser�add_argument�8parse_args)�parser�r�-modelscope/diffdock/scripts/train_diffdock.pyr#srcCsNt|t�rtj�|�St|t�rdd�|��D�St|t�r%dd�|D�S|S)NcSsi|]	\}}|t|��qSr��_resolve_env_vars��.0�k�vrrr�9<dictcomp>,�z%_resolve_env_vars.<locals>.<dictcomp>cSsg|]}t|��qSrr)rr rrr�10<listcomp>.sz%_resolve_env_vars.<locals>.<listcomp>)�11isinstance�str�os�path�12expandvars�dict�items�list)�objrrrr(s131415rcCsDt|ddd��}tt�|�pi�Wd�S1swYdS)N�rzutf-8)�encoding)�openr�yamlZ	safe_load)Zconfig_pathZhandlerrr�load_config4s$�r1cCs8i}|��D]\}}t|t�r|�|�q|||<q|S�N)r*r$r)�update)�configZflat�key�valuerrr�flatten_config9s1617r7cCstdi|��S)Nrr)r4rrr�to_namespaceCsr8cCs,|dvrt�tj��rd�Sd�St�|�S)N>�autoNzcuda:0�cpu)�torch�device�cuda�is_available)Zdevice_namerrr�resolve_deviceGs18r?cCst|d�r19|j��S|��S�N�module)�hasattrrA�20state_dict��modelrrr�state_dict_for_saveMsrFcCst|d�r|jS|Sr@)rBrArDrrr�model_targetQsrGcCs>t�|�tj�|�t�|�tj��rtj�|�dSdSr2)�random�seed�npr;Zmanual_seedr=r>Zmanual_seed_all)rIrrr�set_seedUs212223�rKc24CsV|jsdSzddl}Wnty}ztd�|�d}~ww|j|j|jt|�d�|S)Nrz4wandb is enabled in the config but is not installed.)�project�namer4)�wandb�ImportErrorZinitrL�run_name�vars)�argsrN�excrrr�maybe_init_wandb]s25��rTc26Cs|jdurdSt|j�|j�d�}z@tj|t�d�d�}|jdur-|j|dddd<|�|d�t|�j|dd	d27�|j|d|jd�t	d
|d�WdSt28y�}z(t	d|�tjt|j�dt�d�d�}t|�j|d	d29�t	d�WYd}~dSd}~ww)N�.ptr:�Zmap_location�	optimizer�param_groupsr�lrrET��strict�ema_weights)r<zRestarting from epoch�epoch�	Exception�
best_model.ptz<Due to exception had to take the best epoch and no optimiser)Zrestart_dirrZrestart_ckptr;�loadr<Z30restart_lr�load_state_dictrG�printr^)rRrErWr\Zcheckpoint_path�31checkpointrSrrr�maybe_load_restarths$323334��rdcCs�|jdurdStjt|j�|j�d�t�d�d�}t|t�r+d|vr+d|vr+|d}t|�j	|dd�t35dtt|j�|j�d���dS)	NrUr:rVrErWTrZzUsing pretrained model)Zpretrain_dirr;r`rZ
pretrain_ckptr<r$r)rGrarbr%)rRrErcrrr�maybe_load_pretrain|s36�$rec37sdtt|j|j|j|j|j|jd�}38tj	}|j39dkrtj	nd}|j40dkr&tj	nd}
d}d}d}|jdur6|j41nd}|jdkrN|j
|jdd}td�|��t|�}td�t|j�D�]C}|d	dkrjtd42|j�|jdkr�|d|j
dkr�|d|j
}||jdkr�td�t||||d�\}}n8||jdkr�td
�t||||d�\}}t|��|jd�}n|jdkr�||j
kr�td�t||d||d�\}}i}t|�||j||43||kr�|nd�}td�||d|d|d|d|d|jdd��||k�r|�|���|j�r|�|���t|�|j||44|j �}td�||d|d|d|d|d��|jdu�r~|d|jdk�r~�fdd�tt!|j"�j#�$���D�}t%|||j||��td�|�d�d�d�d ��|�&d!d"���'�D��|j(�r�|jdu�r�|d|jdk�rهfd#d�tt!|j"��$���D�}t%|||j||�}td$�||d|d|d|d ��|�&d%d"�|�'�D��|�&�fd&d"�|�'�D��|j)du�r&|d|j)dk�r&�fd'd�tt!t!|j"d(��j#�$���D�}t%|||j||��td)�|�d�d�d�d ��|�&d*d"���'�D��||k�rE|j�s6|�|���t*�+t,|��}|�-|���nt*�+t,|��}|du�r{|�&d+d"�|�'�D��|�&d,d"�|�'�D��|jdd|d-<|j.||dd.�t,|�}|j/|v�r�|j45dk�r�||j/|k�s�|j46d/k�r�||j/|k�r�||j/}|}t0�1|t2j3�4|d0��||k�r�t0�1|t2j3�4|d1��|j5du�r|j5|v�r|j47dk�r�||j5|
k�s�|j48d/k�r||j5|
k�r||j5}
||k�rt0�1|t2j3�4|d2��|d|k�r(|d}|}t0�1|t2j3�4|d3��||k�r(t0�1|t2j3�4|d4��|j6du�r_|d|j6dk�r_t2j3�4|d3�}t2j3�7|��r_t0�1t0j8|t0�d5�d6�t2j3�4|d7|d�d8���|�r�||k�ss|jdk�rx||j
k�rx|�9�n|jdu�r�|�9|�n|�9|d�t0�1|||�:�|�:�d9�t2j3�4|d:��q[td;�||��td<�||��dS)=N)�	tr_weight�49rot_weight�50tor_weight�51no_torsionZbackbone_weightZsidechain_weight�minrZlayer_linear_warmupr�z'Freezing some parameters until epoch {}zStarting training...�z	Run name:zNew unfreezing step)�step�scheduler_modezUnfreezing all parameters�ZdecayZ
linear_warmupzMoving to plateu scheduler)rmrnrWzZEpoch {}: Training loss {:.4f}  tr {:.4f}   rot {:.4f}   tor {:.4f}   sc {:.4f}  lr {:.4f}ZlossZtr_lossZrot_lossZtor_lossZsidechain_lossrYzQEpoch {}: Validation loss {:.4f}  tr {:.4f}   rot {:.4f}   tor {:.4f}   sc {:.4f}c�g|]}�j�|��qSr��dataset�get�r�i)�52val_loaderrrr#��53��ztrain.<locals>.<listcomp>zcEpoch {}: Val inference rmsds_lt2 {:.3f} rmsds_lt5 {:.3f} min_rmsds_lt2 {:.3f} min_rmsds_lt5 {:.3f}Z	rmsds_lt2Z	rmsds_lt5Z
min_rmsds_lt2Z
min_rmsds_lt5cS�i|]	\}}d||�qS)Zvalinf_rrrrrr!�r"ztrain.<locals>.<dictcomp>csg|]}��|��qSr)rsrt)�val_dataset2rrr#�s��zxEpoch {}: Val inference on second validation rmsds_lt2 {:.3f} rmsds_lt5 {:.3f} min_rmsds_lt2 {:.3f} min_rmsds_lt5 {:.3f}cSrx)Zvalinf2_rrrrrr!r"cs&i|]\}}d||�|d�qS)Zvalinfcomb_rrr)�inf_metricsrrr!s&crprrqrt)�train_loaderrrr#rwi,zeEpoch {}: Train inference rmsds_lt2 {:.3f} rmsds_lt5 {:.3f} min_rmsds_lt2 {:.3f} min_rmsds_lt5 {:.3f}cSrx)Z	traininf_rrrrrr!r"cSrx)Ztrain_rrrrrr!(r"cSrx)Zval_rrrrrr!)r"Z54current_lr)rm�maxzbest_inference_epoch_model.ptz!best_ema_inference_epoch_model.ptz!best_ema_secondary_epoch_model.ptr_zbest_ema_model.ptr:rVr]z_best_model.pt)r]rErWr\z
last_model.ptz#Best Validation Loss {} on Epoch {}z$Best inference metric {} on Epoch {});rr	rfrgrhriZbackbone_loss_weightZsidechain_loss_weight�math�inf�inference_earlystop_goal�val_inference_freq�	schedulerZ55warmup_durZnum_conv_layersrb�formatrT�rangeZn_epochsrPr
r�56parameters�ema_raterr<rX�storeZuse_emaZcopy_tor57Ztest_sigma_intervalsrjZnum_inference_complexesrr�__len__rr3r*Z58double_valZtrain_inference_freq�copyZdeepcopyrFZrestore�logZinference_earlystop_metricr;Zsaver&r'�joinZinference_secondary_metricZsave_model_freq�existsr`rmrC)rRrErWr�r\r{rvr�run_dirryZloss_fnZ
best_val_lossZbest_val_inference_valueZbest_val_secondary_valueZ59best_epochZbest_val_inference_epochZ
freeze_paramsrnZ	wandb_runr]rmZlogsZtrain_lossesZ60val_lossesZinf_datasetZinf_metrics2Zema_state_dictZmodel_state_dictZbest_model_pathr)rzr{ryrvr�train�sv�6162�63��64��	��65�� 66���	(67���	 68���	69707172737475 2"76��77r�cCs�t�}t|j�}t|�}t|�}t|dd�dvrt|j�j|_t	t|dd��|_78t|dd�r9|j79jdkr9dtj
j_tt|d	d80��t|�|jdvsLJ�|jdur^|jdur^|j|jks^J�tj�|j|j�}t|���}t|j81�|d<ttj�|d�|�tt|d
�}t |||j82�\}}}	t!||j83dd�\}84}t"||85|jdur�|jndd�\}}
t#|86�$�|j%d�}t&||87||�t'||88�t(dd�|89�$�D��}t)d|d�t*||90||
||||||	�91dS)NrP>�Nr<r9Zcudnn_benchmarkFr=TrIr>rjr|zmodel_parameters.yml)rR)Zno_parallelrj)rnrocss�|]}|��VqdSr2)�numel)r�prrr�	<genexpr>�s�zmain.<locals>.<genexpr>z92Model withr�)+rr1r4r7r8�getattrrZstemrPr?r<�typer;ZbackendsZcudnnZ	benchmarkrKrrr�r�Zscheduler_patiencer&r'r�Zlog_dirrQr�r%rr�t_to_sigma_complrrr
rr�r�rdre�sumrbr�)ZparsedZ93raw_configZflat_configrRr�Z94saved_argsrr{rvryrE�_rWr�r\r�rrr�mainbs@959697�98r��__main__)1rr�r}r&rH�	functoolsrZpathlibr�typesrZnumpyrJr;r0Z$onescience.datapipes.diffdock.loaderrZ)onescience.utils.diffdock.diffusion_utilsrr�Z"onescience.utils.diffdock.trainingrr	r99rZonescience.utils.diffdock.utilsrr
rZ$onescience.utils.diffdock.validationrZmodels.score_wrapperrrOrrr1r7r8r?rFrGrKrTrdrer�r��__name__rrrr�<module>sN�100
Z+101�