Team Ai
Modelpublic

OneScience-Group/DiffDock

sourceHugging Facemitupdated 2mo agoView on Hugging Face
0likes53downloads
train_diffdock.cpython-311.pyc121 linesDownload Raw Back to __pycache__
1�

2�aOj�>��d�ddlZddlZddlZddlZddlZddlmZddlmZddl	m3Z4ddlZddl
Z
ddlZddlmZddlmZddlmZmZmZmZddlmZmZmZdd	lmZ	d5dlm Z n#e!$r	ddlm Z YnwxYwd�Z"d
�Z#d�Z$d�Z%d�Z&d�Z'd�Z(d�Z)d�Z*d�Z+d�Z,d�Z-d�Z.d�Z/e0dkre/��dSdS)�N)�partial)�Path��SimpleNamespace)�construct_loader)�6t_to_sigma)�inference_epoch_fix�
loss_function�7test_epoch�train_epoch)�ExponentialMovingAverage�get_optimizer_and_scheduler�save_yaml_file)�validate_training_entrypoint�)�build_score_modelc��tj��}|�ddd���|���S)Nz--configTz!Path to the training YAML config.)�required�help)�argparse�ArgumentParser�add_argument�8parse_args)�parsers �B/public/home/liuyx19/modelscope/diffdock/scripts/train_diffdock.pyrr#s>��
�
$�
&�
&�F�9���10�T�8[��\�\�\�������c��t|t��rtj�|��St|t11��rd�|���D��St|t��rd�|D��S|S)Nc�4�i|]\}}|t|����S���_resolve_env_vars��.0�k�vs   r�12<dictcomp>z%_resolve_env_vars.<locals>.<dictcomp>,s'��@�@�@�D�A�q��$�Q�'�'�@�@�@rc�,�g|]}t|����Srr )r#r%s  r�13<listcomp>z%_resolve_env_vars.<locals>.<listcomp>.s!��2�2�2��!�!�$�$�2�2�2r)�14isinstance�str�os�path�15expandvars�dict�items�list)�objs rr!r!(s����#�s���'��w�!�!�#�&�&�&��#�t���A�@�@�C�I�I�K�K�@�@�@�@��#�t���3�2�2�c�2�2�2�2��Jrc��t|dd���5}ttj|��pi��cddd��S#1swxYwYdS)N�rzutf-8)�encoding)�openr!�yaml�	safe_load)�config_path�handles  r�load_configr:4s���	
�k�3��	1�	1�	1�?�V� ����!7�!7�!=�2�>�>�?�?�?�?�?�?�?�?�?�?�?�?����?�?�?�?�?�?s�#A�A�16Ac��i}|���D]5\}}t|t��r|�|���0|||<�6|S�N)r/r)r.�update)�config�flat�key�values    r�flatten_configrB9sY��
�D��l�l�n�n���17��U��e�T�"�"�	��K�K�������D��I�I��Krc��tdi|��S)Nrr)r>s r�to_namespacerDCs���$�$�V�$�$�$rc��|dvr4tjtj���rdnd��Stj|��S)N>N�autozcuda:0�cpu)�torch�device�cuda�is_available)�device_names r�resolve_devicerMGsD���n�$�$��|��18�(?�(?�(A�(A�L�H�H�u�M�M�M��<��$�$�$rc�|�t|d��r|j���n|���S�N�module)�hasattrrP�19state_dict��models r�state_dict_for_saverUMs6��(/��x�(@�(@�X�5�<�"�"�$�$�$�e�FV�FV�FX�FX�Xrc�4�t|d��r|jn|SrO)rQrPrSs r�model_targetrWQs��"�5�(�3�3�>�5�<�<��>rc��tj|��tj�|��tj|��tj���r!tj�|��dSdSr<)�random�seed�nprH�manual_seedrJrK�manual_seed_all)rZs r�set_seedr^Uss��20�K������I�N�N�4����	��d�����z��� � �)�
�21�"�"�4�(�(�(�(�(�)�)rc���|jsdS	ddl}n"#t$r}td��|�d}~wwxYw|�|j|jt|�����|S)Nrz4wandb is enabled in the config but is not installed.)�project�namer>)�wandb�ImportError�initr`�run_name�vars)�argsrb�excs   r�maybe_init_wandbri]s~���:���t�[��������[�[�[��P�Q�Q�WZ�Z�����[����	�J�J�t�|�$�-��T�22�23�J�K�K�K��Ls��24/�*�/c��|j�dSt|j��|j�d�z}	tj|tjd�����}|j�|j|dddd<|�|d��t|���|dd	�25��|�|d|j���td
|d��dS#t$r�}td|��tjt|j��dztjd�����}t|���|d	�26��td��Yd}~dSd}~wwxYw)N�.ptrG��map_location�	optimizer�param_groupsr�lrrTT��strict�ema_weights)rIzRestarting from epoch�epoch�	Exception�
best_model.ptz<Due to exception had to take the best epoch and no optimiser)�restart_dirr�restart_ckptrH�loadrI�27restart_lr�load_state_dictrW�printru)rgrTrnrs�checkpoint_path�28checkpointrhs       r�maybe_load_restartrhs���������4�+�,�,�$�2C�/H�/H�/H�H�O�N��Z��e�l�5�>Q�>Q�R�R�R�29��?�&�?C��J�{�#�N�3�A�6�t�<��!�!�*�[�"9�:�:�:��U���+�+�J�w�,?��+�M�M�M��#�#�J�}�$=�d�k�#�R�R�R�
�%�z�'�':�;�;�;�;�;���N�N�N�
�k�3�����Z��T�%5� 6� 6�� H�W\�Wc�di�Wj�Wj�k�k�k�30��U���+�+�J�t�+�D�D�D�
�L�M�M�M�M�M�M�M�M�M�����	N���s�CC4�431F	�>BF�F	c��|j�dStjt|j��|j�d�ztjd�����}t
|t��rd|vrd|vr|d}t|���	|d���tdtt|j��|j�d�z����dS)	NrkrGrlrTrnTrqzUsing pretrained model)�pretrain_dirrHryr�
pretrain_ckptrIr)r.rWr{r|r*)rgrTr~s   r�maybe_load_pretrainr�|s����� ������T�
���T�%7�"<�"<�"<�<��\�%�(�(����J��*�d�#�#�)��:�(=�(=�+�Q[�B[�B[���(�32�����'�'�33�4�'�@�@�@�	�34"�C��T�->�(?�(?�T�EW�B\�B\�B\�(\�$]�$]�^�^�^�^�^rc35
�6���	��tt|j|j|j|j|j|j���}36tj	}|j37dkrtj	nd}|j38dkrtj	nd}
d}d}d}|j�|j39nd}|jdkr7|j
|jdzzdz40}td�|����t#|��}td��t%|j��D�	]�}|d	zdkrtd41|j��|jdkr�|dz|j
zdkr�|dz|j
z}||jdzkr&td��t+||||���\}}n�||jdzkrMtd
��t+||||���\}}t-|���|j���}n<|jdkr1||j
kr&td��t+||d||���\}}i}t3|�||j||42||kr|nd��}td�||d|d|d|d|d|jdd����||krU|�|�����|jr'|�|�����t?|�|j||43|j ��}td�||d|d|d|d|d����|j��|dz|jzdkr̈fd�t%tC|j"�j#�$������D��}tK|||j||���td�|�d�d�d�d����|�&d ���'��D����|j(�r|j��|dz|jzdkr��	fd!�t%tC|j"�	�$������D��}tK|||j||��}td"�||d|d|d|d����|�&d#�|�'��D����|�&�fd$�|�'��D����|j)��|dz|j)zdkrڈfd%�t%tCtC|j"d&���j#�$������D��}tK|||j||���td'�|�d�d�d�d����|�&d(���'��D����||krw|js'|�|�����tUj+tY|����}|�-|�����n!tUj+tY|����}|��|�&d)�|�'��D����|�&d*�|�'��D����|jdd|d+<|�.||dz�,��tY|��}|j/|vr�|j44dkr||j/|ks|j45d-kr�||j/|kr{||j/}|}taj1|tdj3�4|d.����||kr3taj1|tdj3�4|d/����|j5��|j5|vr~|j46dkr||j5|
ks|j47d-krW||j5|
krF||j5}
||kr3taj1|tdj3�4|d0����|d|krv|d}|}taj1|tdj3�4|d1����||kr3taj1|tdj3�4|d2����|j6��|dz|j6zdkr�tdj3�4|d1��}tdj3�7|��r`taj1taj8|tajd3���4��tdj3�4|d5|dz�d6�����|ri||ks|jdkr ||j
kr|�9��n8|j�|�9|��n|�9|d��taj1|||�:��|�:��d7�tdj3�4|d8�����	��td9�||����td:�||����dS);N)�	tr_weight�48rot_weight�49tor_weight�50no_torsion�backbone_weight�sidechain_weight�minr�layer_linear_warmupr�z'Freezing some parameters until epoch {}zStarting training...�z	Run name:zNew unfreezing step)�step�scheduler_modezUnfreezing all parameters��decay�
linear_warmupzMoving to plateu scheduler)r�r�rnzZEpoch {}: Training loss {:.4f}  tr {:.4f}   rot {:.4f}   tor {:.4f}   sc {:.4f}  lr {:.4f}�loss�tr_loss�rot_loss�tor_loss�sidechain_lossrpzQEpoch {}: Validation loss {:.4f}  tr {:.4f}   rot {:.4f}   tor {:.4f}   sc {:.4f}c�D��g|]}�j�|����Sr��dataset�get)r#�i�51val_loaders  �rr(ztrain.<locals>.<listcomp>�s:��������"�&�&�q�)�)���rzcEpoch {}: Val inference rmsds_lt2 {:.3f} rmsds_lt5 {:.3f} min_rmsds_lt2 {:.3f} min_rmsds_lt5 {:.3f}�	rmsds_lt2�	rmsds_lt5�
min_rmsds_lt2�
min_rmsds_lt5c� �i|]\}}d|z|��S)�valinf_rr"s   rr&ztrain.<locals>.<dictcomp>�s"��J�J�J�d�a���Q���J�J�Jrc�:��g|]}��|����Sr)r�)r#r��val_dataset2s  �rr(ztrain.<locals>.<listcomp>�s7�������� � ��#�#���rzxEpoch {}: Val inference on second validation rmsds_lt2 {:.3f} rmsds_lt5 {:.3f} min_rmsds_lt2 {:.3f} min_rmsds_lt5 {:.3f}c� �i|]\}}d|z|��S)�valinf2_rr"s   rr&ztrain.<locals>.<dictcomp>s"��L�L�L�t�q�!��a���L�L�Lrc�:��i|]\}}d|z|�|zdz��S)�valinfcomb_rr)r#r$r%�inf_metricss   �rr&ztrain.<locals>.<dictcomp>s2���f�f�f���A���*�Q��Q��-?�1�,D�f�f�frc�D��g|]}�j�|����Srr�)r#r��train_loaders  �rr(ztrain.<locals>.<listcomp>s:��������$�(�(��+�+���ri,zeEpoch {}: Train inference rmsds_lt2 {:.3f} rmsds_lt5 {:.3f} min_rmsds_lt2 {:.3f} min_rmsds_lt5 {:.3f}c� �i|]\}}d|z|��S)�	traininf_rr"s   rr&ztrain.<locals>.<dictcomp>s"��L�L�L���1��q��!�L�L�Lrc� �i|]\}}d|z|��S)�train_rr"s   rr&ztrain.<locals>.<dictcomp>(s"��J�J�J�T�Q���A��q�J�J�Jrc� �i|]\}}d|z|��S)�val_rr"s   rr&ztrain.<locals>.<dictcomp>)s"��F�F�F�4�1�a��!��Q�F�F�Fr�52current_lr)r��maxzbest_inference_epoch_model.ptz!best_ema_inference_epoch_model.ptz!best_ema_secondary_epoch_model.ptrvzbest_ema_model.ptrGrlrtz_best_model.pt)rtrTrnrsz
last_model.ptz#Best Validation Loss {} on Epoch {}z$Best inference metric {} on Epoch {});rr53r�r�r�r��backbone_loss_weight�sidechain_loss_weight�math�inf�inference_earlystop_goal�val_inference_freq�	scheduler�54warmup_dur�num_conv_layersr|�formatri�range�n_epochsrerr
�55parameters�ema_raterrIro�store�use_ema�copy_tor�test_sigma_intervalsr��num_inference_complexesr��__len__r	r=r/�56double_val�train_inference_freq�copy�deepcopyrU�restore�log�inference_earlystop_metricrH�saver+r,�join�inference_secondary_metric�save_model_freq�existsryr�rR)rgrTrnr�rsr�r�r�run_dirr��loss_fn�
best_val_loss�best_val_inference_value�best_val_secondary_value�57best_epoch�best_val_inference_epoch�
freeze_paramsr��	wandb_runrtr��logs�train_losses�58val_losses�inf_dataset�inf_metrics2�ema_state_dict�model_state_dict�best_model_pathr�s     ``  `                   @r�trainr��s����������.��?��?��?��1��3����G��H�M�+/�+H�E�+Q�+Q�t�x�x�WX��+/�+H�E�+Q�+Q�t�x�x�WX���J� ���M�6:�6M�6Y�T�2�2�_d�N��~�.�.�.���4�+?�!�+C�D�q�H�
�
�7�>�>�}�M�M�N�N�N� ��&�&�I�	�59 �!�!�!��t�}�%�%�y60�y61���1�9��>�>��+�t�}�-�-�-��>�2�2�2���	�T�_�7T�XY�7Y�7Y��A�I�$�/�1�D��d�*�Q�.�.�.��+�,�,�,�'B����#1�	(�(�(�$�	�9�9���-��1�1�1��1�2�2�2�'B����#1�	(�(�(�$�	�9�7�u�7G�7G�7I�7I�QU�Q^�_�_�_���
�^��
.�
.�5�D�O�3K�3K��.�/�/�/�#>����-�#�$�$�$� �I�y���"�����K��� �=�0�0�K�K�d�62�63��	�h�o�o���V�$��Y�'��Z�(��Z�(��-�.��&�q�)�$�/�
�
�64	65�66	67�68	69��=� � ����e�.�.�0�0�1�1�1��|�
8��#�#�E�$4�$4�$6�$6�7�7�7���z�4�;�70�G�UY�Un�o�o�71�
�_�f�f���6�"��9�%��:�&��:�&��+�,�

�
�		72�		73�		74��"�.�E�A�I��AX�3X�\]�3]�3]������s�4�#?��AS�A[�A[�A]�A]�^�^�_�_����K�.�e�[�$�+�z�[_�`�`�K��u�|�|����,���,���0���0���
�
�
�
�K�K�J�J�k�6G�6G�6I�6I�J�J�J�K�K�K��?�	h�t�6�B��PQ�	�UY�Ul�Gl�pq�Gq�Gq������s�4�#?��AU�AU�AW�AW�X�X�Y�Y����K�/�u�k�4�;�PZ�\`�a�a�L��K�R�R�� ��-� ��-� ��1� ��1���
�
�
�
�K�K�L�L�|�7I�7I�7K�7K�L�L�L�M�M�M��K�K�f�f�f�f�Q]�Qc�Qc�Qe�Qe�f�f�f�g�g�g��$�0�e�a�i�4�C\�5\�`a�5a�5a������s�3�t�'C�S�#I�#I�<�K_�Kg�Kg�Ki�Ki�j�j�k�k����K�.�e�[�$�+�z�[_�`�`�K��w�~�~����,���,���0���0���
�
�
�
�K�K�L�L��8I�8I�8K�8K�L�L�L�M�M�M��=� � ��<�
8��#�#�E�$4�$4�$6�$6�7�7�7�!�]�+>�u�+E�+E�F�F�N����� 0� 0� 2� 2�3�3�3�3�!�]�+>�u�+E�+E�F�F�N�� ��K�K�J�J�\�5G�5G�5I�5I�J�J�J�K�K�K��K�K�F�F�:�3C�3C�3E�3E�F�F�F�G�G�G�!*�!7��!:�4�!@�D����M�M�$�U�Q�Y�M�/�/�/�.�u�5�5���*�d�2�2�
�
*�e�
3�
3��T�=\�8]�ay�8y�8y��-��6�6�4��@_�;`�d|�;|�;|�'+�D�,K�'L�$�',�$��J�'�����g�?^�)_�)_�`�`�`��}�$�$��75�>�2�7�<�<��Ad�+e�+e�f�f�f��*�6�4�;Z�^b�;b�;b�
�
*�e�
3�
3��T�=\�8]�ay�8y�8y��-��6�6�4��@_�;`�d|�;|�;|�'+�D�,K�'L�$��}�$�$��76�>�2�7�<�<��Ad�+e�+e�f�f�f��f���.�.�&�v�.�M��J��J�'�����g��)O�)O�P�P�P��}�$�$��77�>�2�7�<�<��AT�+U�+U�V�V�V���+����d�>R�0R�VW�0W�0W� �g�l�l�7�O�D�D�O��w�~�~�o�.�.�
T��78�5�:�o�E�L�QV�DW�DW�X�X�X�Z\�Za�Zf�Zf�gn�qR�x}�AB�yB�qR�qR�qR�[S�[S�T�T�T��	3��}�$�$���?�)J�)J�u�W[�Wf�Of�Of���� � � � ��(�4����7�8�8�8�8����z�&�1�2�2�2�
�79��)�&�1�1�3�3�*�5�5�7�7�	
�
�
�G�L�L��/�2�2�	80�	81�	82�	83�84�85/�866�876�}�j�88Q�89Q�R�R�R�	�900�917�927�8P�Rj�93k�94k�l�l�l�l�lrc��t��}t|j��}t|��}t	|��}t|dd��dvrt
|j��j|_tt|dd����|_95t|dd��r&|j96jdkrdtj
j_t!t|d	d97����t#|��|jdvsJ�|j�|j�|j|jksJ�t,j�|j|j��}t5|�����}t9|j98��|d<t;t,j�|d��|��t=t>|�
��}tA|||j99��\}}}	tC||j100d���\}101}tE||102|j�|jnd���\}}
tG|103�$��|j%���}tM||104||��tO||105��tQd�|106�$��D����}tSd|d��tU||107||
||||||	�108�109dS)Nre>N�rIrF�cudnn_benchmarkFrJTrZr>r�r�zmodel_parameters.yml)rg)�no_parallelr�)r�r�c3�>K�|]}|���V��dSr<)�numel)r#�ps  r�	<genexpr>zmain.<locals>.<genexpr>�s*����6�6�a����	�	�6�6�6�6�6�6rz110Model withr�)+rr:r>rBrD�getattrr�stemrerMrI�typerH�backends�cudnn�	benchmarkr^rr�r�r��scheduler_patiencer+r,r��log_dirrfr�r*rr�t_to_sigma_complrrrr
r�r�rr��sumr|r�)�parsed�111raw_config�flat_configrgr��112saved_argsrr�r�r�rT�_rnr�rsr�s                r�mainr�bs���
�\�\�F��V�]�+�+�J� ��,�,�K���$�$�D��t�Z��&�&�*�4�4��V�]�+�+�0��
� ���x��!@�!@�A�A�D�K��t�&��.�.�.�4�;�3C�v�3M�3M�)-����&��W�T�6�1�
%�
%�&�&�&� ��&�&�&��(�N�:�:�:�:���*�t�~�/I��&��)@�@�@�@�@��g�l�l�4�<���7�7�G��d�����"�"�J��t�{�+�+�J�x���2�7�<�<��)?�@�@�*�M�M�M��)��5�5�5�J�-=�d�J�PT�P[�-\�-\�*�L�*�l� ��t�{��F�F�F�H�E�1�6��
�8<�8O�8[�t�4�4�af�����I�y�113+�5�+;�+;�+=�+=�T�]�S�S�S�K��t�U�I�{�;�;�;���e�$�$�$��6�6�5�#3�#3�#5�#5�6�6�6�6�6�E�	�,��|�,�,�,�	�$��y�)�[�,�114�T^�`g�iu�v�v�v�v�vr�__main__)1rr�r�r+rY�	functoolsr�pathlibr�typesr�numpyr[rHr6�$onescience.datapipes.diffdock.loaderr�)onescience.utils.diffdock.diffusion_utilsrr��"onescience.utils.diffdock.trainingr	r115rr�onescience.utils.diffdock.utilsr
rr�$onescience.utils.diffdock.validationr�models.score_wrapperrrcrr!r:rBrDrMrUrWr^rirr�r�r��__name__rrr�<module>rs���������������	�	�	�	�
�
�
�
�������������!�!�!�!�!�!�������������A�A�A�A�A�A�T�T�T�T�T�T�����������������������116N�M�M�M�M�M�7�8�8�8�8�8�8�8���7�7�7�6�6�6�6�6�6�6�6�7�������117���?�?�?�118���%�%�%�%�%�%�Y�Y�Y�?�?�?�)�)�)����N�N�N�(119_�120_�121_�Vm�Vm�Vm�r(w�(w�(w�V�z����D�F�F�F�F�F��s�A#�#A1�0A1