Team Ai
Datasetpublic

codekingpro/portable-devtools

sourceHugging Faceupdated 5mo agoView on Hugging Face
1likes14kdownloads
load.cpython-313.pyc58 linesDownload Raw Back to __pycache__
1�

2��j����SSKJr SSKrSSKrSSKrS	SjrS3SjrSSSjjrS
SSjjrSSjr	SSSjjr4g)�)�annotationsNc���SU;a%[USSS9nUR5sSSS5 $URS5(a1SSKnUR	U5nUR5 UR$SSKnURUS5nUR5sSSS5 $!,(df   N�=f![an[S5UeSnAff=f!,(df   g=f)Nz://�rbr��	buffering)zhttp://zhttps://�Oblobfile is not installed. Please install it by running `pip install blobfile`.)5�open�read�6startswith�requests�get�raise_for_status�content�blobfile�ImportError�BlobFile)�blobpath�fr�respr�es      �SD:\code\apps\devtools\python\user_packages\Python313\site-packages\tiktoken/load.py�	read_filers����H��
�(�D�A�
.�!��6�6�8�/�
.����2�3�3���|�|�H�%�������|�|����78�	�	�8�T�	*�a��v�v�x�9+�	*�#/�
.�����]�10��	����11+�	*�s/�B$�4B5�12C�$13B2�514C�?C�C�15C!c�R�[R"U5R5nX!:H$�N)�hashlib�sha256�	hexdigest)�data�
expected_hash�actual_hashs   r�16check_hashr!s"���.�.��&�0�0�2�K��'�'�c��SnS[R;a[RSnO\S[R;a[RSnO4SSKn[RR	UR5S5nSnUS:Xa[
U5$[R"UR55R5n[RR	X55n[RRU5(aQ[USSS	9nUR5nSSS5 Ub[WU5(aW$[R"U5 [
U5n	U(a"[X�5(d[#S17USUS35eSSKn18[R&"USS
9 US-[)U19R+55-S-n[US5nUR-U	5 SSS5 [R."X�5 U	$!,(df   N�=f![ a N�f=f!,(df   NG=f![ a U(aeU	$f=f)NT�TIKTOKEN_CACHE_DIR�DATA_GYM_CACHE_DIRrzdata-gym-cacheF�rrz'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)�exist_ok�.z.tmp�wb)�os�environ�tempfile�path�join�20gettempdirrr�sha1�encoder�existsr	r21r!�remove�OSError�22ValueError�uuid�makedirs�str�uuid4�write�rename)rr�user_specified_cache�	cache_dirr,�	cache_key�23cache_pathrr�contentsr6�tmp_filenames            r�read_file_cachedrB$s������r�z�z�)��J�J�3�4�	�	����	+��J�J�3�4�	���G�G�L�L��!4�!4�!6�8H�I�	�$���B����"�"����X�_�_�.�/�9�9�;�I������i�3�J�	�w�w�~�~�j�!�!�
�*�d�a�
0�A��6�6�8�D�1�� �J�t�]�$C�$C��K�	��I�I�j�!���"�H��Z��@�@��5�h�Z�{�=�/�ZH�
I�24�	25�26�	�27���I��-�!�C�'�#�d�j�j�l�*;�;�f�D��
�,��
%��
�G�G�H��&�28�	�	�,�+��O�?1�
0���	��	��&�
%������ ��O��sI�H�H�AH>�H-�,H>�29H�30H*�)H*�-31H;�7H>�>I�Ic�^^�[S5Vs/sH6n[U5R5(dM#[U5S:wdM4UPM8 nnUVs0sHn[U5U_M snmSn[S5H1nXV;dM32URU5 UT[SU-5'US-
nM3 [	U5S:Xde[X5R
5nURS5SSV	s/sHn	[U	R55PM n33n	S34U4Sjjn[U5VVs0sHup�[U/5U_M n
nnA[	U
5nU35Hup�X}U"U5U"U5-'US-
nM SSKnUR[X55nUR5VVs0sHunnU"U5U_M nnnURSS5 URS	S5 U(a"UHn[	U5S:XdMUUU
U'M U
U:XdeU
$s snfs snfs sn	fs snnfs snnf)N�� r��36�����c�.>�[U4SjU55$)Nc3�.># �UH37nTUv� M g7fr�)�.0�b�data_gym_byte_to_bytes  �r�	<genexpr>�Kdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>rs����=�u�!�*�1�-�u�s�)�bytes)�valuerNs �r�decode_data_gym�8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gymqs����=�u�=�=�=r"s
<|endoftext|>s<|startoftext|>)rRr8�returnrQ)�range�chr�isprintable�append�lenrB�decode�split�tuple�	enumeraterQ�json�loads�items�pop)�vocab_bpe_file�encoder_json_file�vocab_bpe_hash�encoder_json_hash�clobber_one_byte_tokensrM�rank_to_intbyte�n�vocab_bpe_contents�	merge_str�38bpe_mergesrS�i�	bpe_ranks�first�secondr_�encoder_json�k�v�encoder_json_loadedrNs                     @r�data_gym_to_mergeable_bpe_ranksruZs(���#(��+�X�+�Q��Q��1C�1C�1E�q�#�a�&�TW�-�q�+�O�X�0?�@��1�S��V�Q�Y��@��	�A�
�4�[���#��"�"�1�%�34�!�#�d�Q�h�-�0�
��F�A�	�39���4�'�'�'�*�.�I�P�P�R��<N�<T�<T�UY�<Z�[\�]_�<`�a�<`�y�%�	���)�*�<`�J�a�>�40,5�_�+E�F�+E�4�1���s��Q��+E�I�F��	�I��A�#�
��FG�/�%�(�?�6�+B�B�C�	�Q���$��41�:�:�.�/@�T�U�L�=I�=O�=O�=Q�R�=Q�T�Q��?�1�-�q�0�=Q��R����,�d�3����.��5��$�A��1�v��{�2�1�5�	�!��%��+�+�+�+����_Y��@��b��G��Ss(�"H�
H�H�H�-#H�(H#�$H)c�p�SSKnURUS5n[UR	5SS9HJupVUR[R"U5S-[U5R5-S-5 ML SSS5 g![an[S5UeSnAff=f!,(df   g=f)Nrrr)c��US$)NrFrK)�xs r�<lambda>�#dump_tiktoken_bpe.<locals>.<lambda>�s��1�Q�4r")�key� �42)43rrr�sortedrar:�base64�	b64encoder8r1)rn�tiktoken_bpe_filerrr�token�ranks       r�dump_tiktoken_bper��s�����4445�	�	�,�d�	3�q�!�)�/�/�"3��H�K�E�
�G�G�F�$�$�U�+�d�2�S��Y�5E�5E�5G�G�%�O�P�I�464�	3��	���]�47��	����484�	3�s#�B	�A(B'�	49B$�B�B$�'50B5c��[X5n0nUR5HAnU(dMUR5upV[U5U[R51"U5'MC U$![an[SU<SU35UeSnAff=f)NzError parsing line z in )rB�52splitlinesr\�intr�	b64decode�	Exceptionr5)r�rr@�ret�liner�r�rs        r�load_tiktoken_bper��s���� 1�A�H�53�C��#�#�%����	[��*�*�,�K�E�+.�t�9�C�� � ��'�(�&��J���	[��2�4�(�$�?P�>Q�R�S�YZ�Z��	[�s�4A%�%54B�/B�B)rr8rUrQ)rrQrr8rU�boolr)rr8r�55str | NonerUrQ)NNF)rcr8rdr8rer�rfr�rgr�rU�dict[bytes, int])rnr�r�r8rU�None)r�r8rr�rUr�)�56__future__rrrr*rr!rBrur�r�rKr"r�<module>r�su��"�
��	��.(�573�r"&�$(�$)�7��7��7��7�"�	7�58"�7��
7�t	Q�r"
codekingpro/portable-devtools · Team Ai