Team Ai
Modelpublic

hymenjj/llama-cpp-python-prebuilt

sourceHugging Faceupdated 7mo agoView on Hugging Face
0likes
_internals.cpython-311.pyc158 linesDownload Raw Back to __pycache__
1�

200�izs���ddlmZddlZddlZddlmZmZmZmZm	Z	m3Z4mZddlm
Z
mZddlmZddlZddlmZddlTddlmZdd	lmZddlmZGd5�d��ZGd�d
��ZGd�d��ZGd�d��Zd�Z e
Gd�d����Z!e
Gd�d����Z"Gd�d��Z#Gd�d��Z$dS)�)�annotationsN)�Dict�List�Tuple�Optional�Sequence�Callable�Union)�	dataclass�field)�	ExitStack�)�*)�LlamaGrammar)�suppress_stdout_stderrc��eZdZdZdd�d8d6�Zd�Zd�Zd9d�Zd9d�Zd9d�Z	d9d�Z7d:d�Zd;d�Zd9d�Z
d9d�Zd<d�Zd=d�Zd>d�Zd?d�Zd9d�Zd9d �Zd9d!�Zd9d"�Zd9d#�Zd9d$�Zd9d%�Zd9d&�Zd9d'�Zd@d(�Zd@d)�ZdAd.�ZdBdCd0�ZdBdDd3�Z dEd5�Z!e"d6���Z#d7S)F�8LlamaModelz�Intermediate Python wrapper for a llama.cpp llama_model.9    NOTE: For stability it's recommended you use the Llama class instead.T��verbose�10path_model�str�params�llama_cpp.llama_model_paramsr�boolc�R��|�_|�_|�_t���_d}t11j�|��std|�����t|���5tj�j�d���j��}ddd��n#1swxYwY|�td|�����tj
|��}|�td|�����|�_|�_d�_�fd�}�j�|��dS)NzModel path does not exist: )�disable�utf-8z Failed to load model from file: z Failed to get vocab from model: c�Z���j�dStj�j��d�_dS�N)�model�	llama_cpp�llama_model_free��selfs��?/usr/local/lib/python3.11/site-packages/llama_cpp/_internals.py�12free_modelz'LlamaModel.__init__.<locals>.free_modelE�.����z�!����&�t�z�2�2�2��D�J�J�J�)rrrr
�_exit_stack�os�path�exists�13ValueErrorrr!�llama_model_load_from_file�encode�llama_model_get_vocabr �vocab�sampler�callback)r$rrrr r1r&s`      r%�__init__zLlamaModel.__init__#s���%���������$�;�;������w�~�~�j�)�)�	I��G�:�G�G�H�H�H�
#�G�
4�
4�
4�	�	��8���&�&�w�/�/�����E�	�	�	�	�	�	�	�	�	�	�	����	�	�	�	�14�=��L�15�L�L�M�M�M��/��6�6���=��L�16�L�L�M�M�M���17���18����	�	�	�	�	�	
��!�!�*�-�-�-�-�-s�-3B,�,B0�3B0c���|j�Mt|j��D]\}}tj|j|��� |j���|j���dSr)r2�reversed�custom_samplersr!�llama_sampler_chain_remove�clearr)�close)r$�i�_s   r%r:zLlamaModel.closeMsr���<�#� ��!5�6�6�
F�
F���1��4�T�\�1�E�E�E�E�� �&�&�(�(�(����� � � � � r(c�.�|���dSr�r:r#s r%�__del__zLlamaModel.__del__U����19�20�����r(�return�intc�4�tj|j��Sr)r!�llama_vocab_typer1r#s r%�21vocab_typezLlamaModel.vocab_typeX����)�$�*�5�5�5r(c�4�tj|j��Sr)r!�llama_vocab_n_tokensr1r#s r%�n_vocabzLlamaModel.n_vocab[����-�d�j�9�9�9r(c�4�tj|j��Sr)r!�llama_model_n_ctx_trainr r#s r%�n_ctx_trainzLlamaModel.n_ctx_train^����0���<�<�<r(c�4�tj|j��Sr)r!�llama_model_n_embdr r#s r%�n_embdzLlamaModel.n_embdas���+�D�J�7�7�7r(�floatc�4�tj|j��Sr)r!�!llama_model_rope_freq_scale_trainr r#s r%�rope_freq_scale_trainz LlamaModel.rope_freq_scale_trainds���:�4�:�F�F�Fr(c��tjd��}tj|j|d��|j�d��S)N�r)�ctypes�create_string_bufferr!�llama_model_descr �value�decode)r$�bufs  r%�desczLlamaModel.descgs?���)�$�/�/���"�4�:�s�D�9�9�9��y����(�(�(r(c�4�tj|j��Sr)r!�llama_model_sizer r#s r%�sizezLlamaModel.sizelrFr(c�4�tj|j��Sr)r!�llama_model_n_paramsr r#s r%�n_paramszLlamaModel.n_paramsorJr(�name�ctypes.c_void_pc� �td���)Nz*get_tensor is not implemented in llama.cpp��NotImplementedError)r$res  r%�22get_tensorzLlamaModel.get_tensorrs��!�"N�O�O�Or(�tokenc�\�tj|j|���d��S�Nr)r!�llama_vocab_get_textr1r\�r$rks  r%�token_get_textzLlamaModel.token_get_textws%���-�d�j�%�@�@�G�G��P�P�Pr(c�6�tj|j|��Sr)r!�llama_vocab_get_scorer1ros  r%�token_get_scorezLlamaModel.token_get_scorezs���.�t�z�5�A�A�Ar(c�6�tj|j|��Sr)r!�llama_vocab_get_attrr1ros  r%�token_get_attrzLlamaModel.token_get_attr}s���-�d�j�%�@�@�@r(c�4�tj|j��Sr)r!�llama_vocab_bosr1r#s r%�	token_boszLlamaModel.token_bos�����(���4�4�4r(c�4�tj|j��Sr)r!�llama_vocab_eosr1r#s r%�	token_eoszLlamaModel.token_eos�rzr(c�4�tj|j��Sr)r!�llama_vocab_clsr1r#s r%�	token_clszLlamaModel.token_cls�rzr(c�4�tj|j��Sr)r!�llama_vocab_sepr1r#s r%�	token_sepzLlamaModel.token_sep�rzr(c�4�tj|j��Sr)r!�llama_vocab_nlr1r#s r%�token_nlzLlamaModel.token_nl�s���'��23�3�3�3r(c�4�tj|j��Sr)r!�llama_vocab_fim_prer1r#s r%�token_prefixzLlamaModel.token_prefix�����,�T�Z�8�8�8r(c�4�tj|j��Sr)r!�llama_vocab_fim_midr1r#s r%�token_middlezLlamaModel.token_middle�r�r(c�4�tj|j��Sr)r!�llama_vocab_fim_sufr1r#s r%�token_suffixzLlamaModel.token_suffix�r�r(c�4�tj|j��Sr)r!�llama_vocab_eotr1r#s r%�	token_eotzLlamaModel.token_eot�rzr(c�4�tj|j��Sr)r!�llama_vocab_get_add_bosr1r#s r%�
add_bos_tokenzLlamaModel.add_bos_token�rNr(c�4�tj|j��Sr)r!�llama_vocab_get_add_eosr1r#s r%�
add_eos_tokenzLlamaModel.add_eos_token�rNr(�text�bytes�add_bos�specialc	���|���}tj|z��}tj|j|t|��||||��}|dkrlt
|��}tj|z��}tj|j|t|��||||��}|dkrtd|�d|�����t|d|���S)NrzFailed to tokenize: text="z" n_tokens=)	rMr!�llama_token�llama_tokenizer1�len�abs�RuntimeError�list)r$r�r�r��n_ctx�tokens�n_tokenss       r%�tokenizezLlamaModel.tokenize�s���� � �"�"���'�%�/�2�2���+��J��c�$�i�i�����24�25���a�<�<��8�}�}�H��+�h�6�9�9�F� �/��26�D�#�d�)�)�V�X�w����H��!�|�|�"�L��L�L�(�L�L�����F�9�H�9�%�&�&�&r(Fc��tjd��}tj|j||dd|��t|��S)N� r)rXrYr!�llama_token_to_piecer1r�)r$rkr�r]s    r%�token_to_piecezLlamaModel.token_to_piece�s;���)�"�-�-���&�t�z�5�#�r�1�g�N�N�N��S�z�z�r(r��	List[int]c	�z�d}d}tj|z��}|D]T}tj|jtj|��||d|��}||ksJ�|t
|d|���z
}�Ut|��dkr6|d|���kr|dd�dkr27|dd�n|S)Nr(r�rr� )	rX�c_charr!r�r1r�r�r�ry)r$r�r��outputra�bufferrk�ns        r%�28detokenizezLlamaModel.detokenize�s��������-�$�&�)�)���	(�	(�E��.��29�I�1�%�8�8�&�$��7���A���9�9�9�9��e�F�2�A�2�J�'�'�'�F�F�30�6�{�{�Q���6�!�9����0@�0@�#@�#@�V�A�a�C�[�TX�EX�EX�
�1�2�2�J�J��	31r(�Dict[str, str]c�j�i}d}tj|��}d|z|_tt	j|j����D]�}t	j|j|||��}||kr5|dz}tj|��}t	j|j|||��}|j�d��}t	j	|j|||��}||kr5|dz}tj|��}t	j	|j|||��}|j�d��}|||<��|S)NrW�rr)32rXrYr[�ranger!�llama_model_meta_countr �llama_model_meta_key_by_indexr\�!llama_model_meta_val_str_by_index)r$�metadata�buffer_sizer�r;�nbytes�keyr[s        r%r�zLlamaModel.metadata�sJ��#%�����,�[�9�9���{�*����y�7��33�C�C�D�D�	"�	"�A��<��34�A�v�{���F���#�#�$�q�j���4�[�A�A��"�@��J��6�;�����,�%�%�g�.�.�C��@��35�A�v�{���F���#�#�$�q�j���4�[�A�A��"�D��J��6�;�����L�'�'��0�0�E�!�H�S�M�M��r(c�(�tj��S)z#Get the default llama_model_params.)r!�llama_model_default_params�r(r%�default_paramszLlamaModel.default_params�s���3�5�5�5r(N)rrrrrr�rArB)rArR)rAr)rerrArf)rkrBrAr)rkrBrArR)rkrBrArB)rAr)r�r�r�rr�r)F)rkrBr�rrAr�)r�r�r�rrAr�)rAr�)$�__name__�36__module__�__qualname__�__doc__r4r:r?rErIrMrQrUr^rardrjrprsrvryr}r�r�r�r�r�r�r�r�r�r�r�r�r��staticmethodr�r�r(r%rrs�������M�M��(.�(.�(.�(.�(.�(.�T!�!�!����6�6�6�6�:�:�:�:�=�=�=�=�8�8�8�8�G�G�G�G�)�)�)�)�376�6�6�6�:�:�:�:�P�P�P�P�38Q�Q�Q�Q�B�B�B�B�A�A�A�A�395�5�5�5�5�5�5�5�5�5�5�5�5�5�5�5�4�4�4�4�9�9�9�9�9�9�9�9�9�9�9�9�5�5�5�5�=�=�=�=�=�=�=�=�40'�'�'�'�$�����4142�43�44�45�46�&����>�6�6��\�6�6�6r(rc�8�eZdZdZdd�dRd47�Zd�Zd�ZdSd�ZdSd�Zd�Z	dTd�Z48dUd�ZdVd�ZdWd�Z
dSd�ZdXd�ZdXd �ZdYd#�Zd$�ZdZd&�Zd'�ZdZd(�ZdVd)�Zd[d+�Zd\d5�Zd]d6�Zd^d9�Zd_d;�Zd_d<�Zd_d=�Zd`d?�ZdadB�ZdbdH�Z dcdI�Z!dddJ�Z"dddK�Z#dedM�Z$dN�Z%dO�Z&e'dP���Z(dQS)f�LlamaContextz�Intermediate Python wrapper for a llama.cpp llama_context.49    NOTE: For stability it's recommended you use the Llama class instead.Trr rr�llama_cpp.llama_context_paramsrrc�X��|�_|�_|�_t���_tj�jj�j��}|�td���|�_tj	�j���_50d�_�fd�}�j�|��dS)NzFailed to create llama_contextc�Z���j�dStj�j��d�_dSr)�ctxr!�51llama_freer#s�r%�free_ctxz'LlamaContext.__init__.<locals>.free_ctx
s.����x����� ���*�*�*��D�H�H�Hr()
r rrr
r)r!�llama_init_from_modelr-r��llama_get_memory�memoryr2r3)r$r rrr�r�s`     r%r4zLlamaContext.__init__�s������52�������$�;�;����-�d�j�.>���L�L���;��=�>�>�>�����0���:�:������	�	�	�	�	�	
��!�!�(�+�+�+�+�+r(c�8�|j���dSr�r)r:r#s r%r:zLlamaContext.close������� � � � � r(c�.�|���dSrr>r#s r%r?zLlamaContext.__del__r@r(rArBc�4�tj|j��Sr)r!�llama_n_ctxr�r#s r%r�zLlamaContext.n_ctxs���$�T�X�.�.�.r(c�4�tj|j��Sr)r!�llama_pooling_typer�r#s r%�pooling_typezLlamaContext.pooling_types���+�D�H�5�5�5r(c�\�|j�53Jd���tj|jd��dS)N�Memory is not initializedT)r�r!�llama_memory_clearr#s r%�kv_cache_clearzLlamaContext.kv_cache_clear!s4���{�&�&�(C�&�&�&��$�T�[�$�7�7�7�7�7r(�seq_id�p0�p1c�t�|j�54Jd���|dkr|nd}tj|j|||��dS)Nr�r)r�r!�llama_memory_seq_rm)r$r�r�r�s    r%�kv_cache_seq_rmzLlamaContext.kv_cache_seq_rm%sI���{�&�&�(C�&�&�&�!�Q�;�;���A���%�d�k�6�2�r�B�B�B�B�Br(�55seq_id_src�56seq_id_dstc�b�|j�57Jd���tj|j||||��dS�Nr�)r�r!�llama_memory_seq_cp)r$r�r�r�r�s     r%�kv_cache_seq_cpzLlamaContext.kv_cache_seq_cp*s:���{�&�&�(C�&�&�&��%�d�k�:�z�2�r�R�R�R�R�Rr(c�\�|j�58Jd���tj|j|��dSr�)r�r!�llama_memory_seq_keep�r$r�s  r%�kv_cache_seq_keepzLlamaContext.kv_cache_seq_keep.s4���{�&�&�(C�&�&�&��'���V�<�<�<�<�<r(�shiftc�b�|j�59Jd���tj|j||||��dSr�)r�r!�llama_memory_seq_add)r$r�r�r�r�s     r%�kv_cache_seq_shiftzLlamaContext.kv_cache_seq_shift2s:���{�&�&�(C�&�&�&��&�t�{�F�B��E�J�J�J�J�Jr(c�4�tj|j��Sr)r!�llama_state_get_sizer�r#s r%�get_state_sizezLlamaContext.get_state_size6����-�d�h�7�7�7r(�batch�60LlamaBatchc�t�tj|j|j��}|dkrt	d|�����dS)Nrzllama_decode returned )r!�llama_decoder�r�r��r$r��return_codes   r%r\zLlamaContext.decodeA�J���,��H��K�61�62���!����E��E�E�F�F�F��r(c�t�tj|j|j��}|dkrt	d|�����dS)Nrzllama_encode returned )r!�llama_encoder�r�r�r�s   r%r/zLlamaContext.encodeIrr(�	n_threads�n_threads_batchc�<�tj|j||��dSr)r!�llama_set_n_threadsr�)r$rrs   r%�
set_n_threadszLlamaContext.set_n_threadsQs���%�d�h�	�?�K�K�K�K�Kr(c�4�tj|j��Sr)r!�llama_get_logitsr�r#s r%�63get_logitszLlamaContext.get_logitsTs���)�$�(�3�3�3r(r;c�6�tj|j|��Sr)r!�llama_get_logits_ithr��r$r;s  r%�get_logits_ithzLlamaContext.get_logits_ithWs���-�d�h��:�:�:r(c�4�tj|j��Sr)r!�llama_get_embeddingsr�r#s r%�get_embeddingszLlamaContext.get_embeddingsZr�r(c�6�tj|j|��Sr)r!�llama_get_embeddings_ithr�r
s  r%�get_embeddings_ithzLlamaContext.get_embeddings_ith]s���1�$�(�A�>�>�>r(c�6�tj|j|��Sr)r!�llama_get_embeddings_seqr�r�s  r%�get_embeddings_seqzLlamaContext.get_embeddings_seq`s���1�$�(�F�C�C�Cr(�seedc� �td���)Nz4set_rng_seed is deprecated, use LlamaSampler insteadrh)r$rs  r%�set_rng_seedzLlamaContext.set_rng_seede���!�"X�Y�Y�Yr(�64candidates�'_LlamaTokenDataArray'�last_tokens_data�('llama_cpp.Array[llama_cpp.llama_token]'�penalty_last_n�penalty_repeatrR�penalty_freq�penalty_presentc� �td���)NzCsample_repetition_penalties is deprecated, use LlamaSampler insteadrh)r$rrr r!r"r#s       r%�sample_repetition_penaltiesz(LlamaContext.sample_repetition_penaltieshs��"�"g�h�h�hr(c� �td���)Nz6sample_softmax is deprecated, use LlamaSampler insteadrh�r$rs  r%�sample_softmaxzLlamaContext.sample_softmaxs���!�"Z�[�[�[r(�k�min_keepc� �td���)Nz4sample_top_k is deprecated, use LlamaSampler insteadrh)r$rr*r+s    r%�sample_top_kzLlamaContext.sample_top_kvrr(�pc� �td���)Nz4sample_top_p is deprecated, use LlamaSampler insteadrh�r$rr.r+s    r%�sample_top_pzLlamaContext.sample_top_pyrr(c� �td���)Nz4sample_min_p is deprecated, use LlamaSampler insteadrhr0s    r%�sample_min_pzLlamaContext.sample_min_p|rr(c� �td���)Nz6sample_typical is deprecated, use LlamaSampler insteadrhr0s    r%�sample_typicalzLlamaContext.sample_typicals��"�"Z�[�[�[r(�tempc� �td���)Nz3sample_temp is deprecated, use LlamaSampler insteadrh)r$rr6s   r%�sample_tempzLlamaContext.sample_temp�s��!�"W�X�X�Xr(�grammarrc� �td���)Nz6sample_grammar is deprecated, use LlamaSampler insteadrh)r$rr9s   r%�sample_grammarzLlamaContext.sample_grammar�r)r(�tau�eta�m�mu�,llama_cpp.CtypesPointerOrRef[ctypes.c_float]c� �td���)Nz=sample_token_mirostat is deprecated, use LlamaSampler insteadrh)r$rr<r=r>r?s      r%�sample_token_mirostatz"LlamaContext.sample_token_mirostat�s��"�"a�b�b�br(c� �td���)Nz@sample_token_mirostat_v2 is deprecated, use LlamaSampler insteadrh)r$rr<r=r?s     r%�sample_token_mirostat_v2z%LlamaContext.sample_token_mirostat_v2�s��"�"d�e�e�er(c� �td���)Nz;sample_token_greedy is deprecated, use LlamaSampler insteadrhr's  r%�sample_token_greedyz LlamaContext.sample_token_greedy�s��!�"_�`�`�`r(c� �td���)Nz4sample_token is deprecated, use LlamaSampler insteadrhr's  r%�sample_tokenzLlamaContext.sample_token�rr(rkc� �td���)Nz<grammar_accept_token is deprecated, use LlamaSampler insteadrh)r$r9rks   r%�grammar_accept_tokenz!LlamaContext.grammar_accept_token�s��!�"`�a�a�ar(c�8�tj|j��dSr)r!�llama_perf_context_resetr�r#s r%�
reset_timingszLlamaContext.reset_timings�����*�4�8�4�4�4�4�4r(c�8�tj|j��dSr)r!�llama_perf_context_printr�r#s r%�
print_timingszLlamaContext.print_timings�rNr(c�(�tj��S)z%Get the default llama_context_params.)r!�llama_context_default_paramsr�r(r%r�zLlamaContext.default_params�s���5�7�7�7r(N)r rrr�rrr�)r�rBr�rBr�rB)r�rBr�rBr�rBr�rB)r�rB)r�rBr�rBr�rBr�rB)r�r�)rrBrrB)r;rB�rrB)rrrrr rBr!rRr"rRr#rR)rr)rrr*rBr+rB)rrr.rRr+rB)rrr6rR)rrr9r)rrr<rRr=rRr>rBr?r@rArB)65rrr<rRr=rRr?r@rArB)rrrArB)r9rrkrB))r�r�r�r�r4r:r?r�r�r�r�r�r�r�r�r\r/rr66rrrrrr%r(r-r1r3r5r8r;rBrDrFrHrJrMrQr�r�r�r(r%r�r��s7������M�M��,�,�,�,�,�,�:!�!�!����/�/�/�/�6�6�6�6�8�8�8�C�C�C�C�67S�S�S�S�=�=�=�=�K�K�K�K�8�8�8�8�G�G�G�G�G�G�G�G�L�L�L�L�4�4�4�;�;�;�;�8�8�8�?�?�?�?�D�D�D�D�68Z�Z�Z�Z�	i�	i�	i�	i�\�\�\�\�Z�Z�Z�Z�Z�Z�Z�Z�Z�Z�Z�Z�\�\�\�\�69Y�Y�Y�Y�\�\�\�\�c�c�c�c�f�f�f�f�a�a�a�a�Z�Z�Z�Z�b�b�b�b�5�5�5�5�5�5��8�8��\�8�8�8r(r�c�F�eZdZdd�dd	�Zd70�Zd�Zdd
�Zd�Zdd�Zdd�Z	dS)r�Trr�rB�embd�	n_seq_maxrrc�,��|�_|�_|�_|�_t	���_t
j�j�j�j��}|�td���|�_	d�_71�fd�}�j�|��dS)NzFailed to create llama_batchc�Z���j�dStj�j��d�_dSr)r�r!�llama_batch_freer#s�r%�72free_batchz'LlamaBatch.__init__.<locals>.free_batch�r'r()�	_n_tokensrVrWrr
r)r!�llama_batch_initr-r�r2r3)r$r�rVrWrr�r[s`      r%r4zLlamaBatch.__init__�s����"�����	�"������$�;�;����*�4�>�4�9�d�n�U�U���=��;�<�<�<���73����	�	�	�	�	�	
��!�!�*�-�-�-�-�-r(c�8�|j���dSrr�r#s r%r:zLlamaBatch.close�r�r(c�.�|���dSrr>r#s r%r?zLlamaBatch.__del__�r@r(rAc��|jjSr�r�r�r#s r%r�zLlamaBatch.n_tokens�s
���z�"�"r(c��d|j_dS)Nrrar#s r%�resetzLlamaBatch.reset�s����74���r(r��
Sequence[int]�n_past�75logits_allc�8�t|��}||j_t|��D]\}|||jj|<||z|jj|<d|jj|d<d|jj|<||jj|<�]d|jj|dz76<dS�NrrT�	r�r�r�r�rk�posr��n_seq_id�logits)r$r�rerfr�r;s      r%�	set_batchzLlamaBatch.set_batch�s����u�:�:��&��77���x���	.�	.�A�"'��(�D�J��Q�� &��78�D�J�N�1��&'�D�J��a� ��#�%&�D�J���"�#-�D�J��a� � �*.��79��(�Q�,�'�'�'r(r�c�f�t|��}|jj}|jxj|z
c_t|��D]^}||z}|||jj|<||jj|<||jj|d<d|jj|<||jj|<�_d|jj|dz80<dSrhri)r$r�r�rfr��	n_tokens0r;�js        r%�add_sequencezLlamaBatch.add_sequence�s����u�:�:���J�'�	��81���x�'����x���	.�	.�A��A�
�A�"'��(�D�J��Q�� !�D�J�N�1��&,�D�J��a� ��#�%&�D�J���"�#-�D�J��a� � �*.��82��(�Q�,�'�'�'r(N)r�rBrVrBrWrBrrr�)r�rdrerBrfr)r�rdr�rBrfr)83r�r�r�r4r:r?r�rcrmrqr�r(r%r�r��s�������KO�.�.�.�.�.�.�2!�!�!����#�#�#�#� � � �	/�	/�	/�	/�/�/�/�/�/�/r(r�c��eZdZdd�Zd	d�ZdS)84�LlamaTokenDataArrayrIrBc	��||_tj|jftjdtjfdtjfdtjfgd������|_tj|jj	�85tj��|jd���|_tj
|jtj���|_tj|jtj���|_d|_dS)	N�id�logitr.T)�align)�dtypeF)�datara�sorted)rI�np�recarrayrx�intc�single�candidates_datar!�llama_token_data_arrayrX�data_as�llama_token_data_pr�arange�default_candidates_data_id�zeros�default_candidates_data_pr2)r$rIs  r%r4zLlamaTokenDataArray.__init__�s������!�{�
�\�O��(�����7�B�I�"6��b�i�8H�I�QU���� 86� 87� 88���$�:��%�,�4�4�Y�5Q�R�R����89�90�91���92+-�)�D�L���*P�*P�*P��'�)+��$�,�b�i�)P�)P�)P��&�����r(rl�npt.NDArray[np.single]c��|j|jjdd�<||jjdd�<|j|jjdd�<d|j_|j|j_	dS)NF)93r�rrurvr�r.rrzrIra)r$rls  r%�copy_logitszLlamaTokenDataArray.copy_logitss`��%)�%D�������"�(.���"�1�1�1�%�$(�$B����q�q�q�!�!&����#�|�����r(N)rIrB)rlr�)r�r�r�r4r�r�r(r%rsrs�s<����������",�,�,�,�,�,r(rsc���ttj�|������dkr|S�fd�|D��S)N�c���g|]}|�z��Sr�r�)�.0�v�norms  �r%�94<listcomp>z'normalize_embedding.<locals>.<listcomp>s���(�(�(��A��H�(�(�(r()rRr{�linalgr�)�	embeddingr�s @r%�normalize_embeddingr�sG��������	�*�*�+�+�D��s�{�{���(�(�(�(�i�(�(�(�(r(c�<�eZdZUdZded<dZded<dZded<dZd	ed95<dZd	ed<d
Z	d	ed<d
Z96d	ed<dZd	ed<dZded<d
Z
d	ed<dZd	ed<dZd	ed<dZded<dZd	ed<dZd	ed<dZded<dZd ed!<dZd ed"<d
Zd	ed#<ee�$��Zd%ed&<d'S)(�LlamaSamplingParams�@rB�n_prevr�n_probs�(�top_kgffffff�?rR�top_pg�������?�min_pg�?�tfs_z�	typical_pg�������?r6r r!r�r"r#�mirostatg@�mirostat_taug�������?�mirostat_etaTr�penalize_nl�rr9�cfg_negative_prompt�	cfg_scale��default_factoryzdict[int, float]�97logit_biasN)r�r�r�r��__annotations__r�r�r�r�r�r�r6r r!r"r#r�r�r�r�r9r�r�r�dictr�r�r(r%r�r�sp��������F������G������E�O�O�O�O��E������E������E������I������D������N������N������L�����!�O�!�!�!�!��H������L������L������K������G�����!��!�!�!�!��I�����#(�5��#>�#>�#>�J�>�>�>�>�>�>r(r�c���eZdZUee���Zded<eej���Z	ded<dZ98ded<ee���Zd	ed99<ee���Z
ded<d
�Zd�Zd!d�Zd"d�Z		d#d$d�Zd%d �ZdS)&�LlamaSamplingContextr�r�rzctypes.c_float�mirostat_muNzOptional[LlamaGrammar]r9z	list[int]�prevz list[llama_cpp.llama_token_data]�curc�f�g|_g|_|j�|j���dSdSr)r�r�r9rcr#s r%rczLlamaSamplingContext.reset>s:����	�����<�#��L��� � � � � �$�#r(c��t|j|j|j|j���|j������S)N)rr�r9r�r�)r�rr�r9r��copyr�r#s r%�cpzLlamaSamplingContext.cpDsE��#��;��(��L�����!�!���
�
���100�101�102�	103r(rA�
Optional[int]c�P�t|j��dkr
|jdSdS)Nr�����)r�r�r#s r%�lastzLlamaSamplingContext.lastMs&���t�y�>�>�A����9�R�=� ��4r(�ctx_mainr�r�rBrc�x�|j�|j|d����d��Srm)r r�r�r\)r$r�r�s   r%�prev_strzLlamaSamplingContext.prev_strSs2���~�(�(���A�2�3�3��8�8�?�?��H�H�Hr(r�idx�logits_array� Optional[npt.NDArray[np.single]]c� �td���)NzCLlamaSamplingContext.sample is deprecated, use LlamaSampler insteadrh)r$r�r�r�s    r%�samplezLlamaSamplingContext.sampleVs��"�"g�h�h�hr(ru�
apply_grammarrc�:�|j�|��dSr)r��append)r$r�rur�s    r%�acceptzLlamaSamplingContext.accept_s���	��������r()rAr�)r�r�r�rBrAr)rN)r�r�r�rBr�r�)r�r�rurBr�r)r�r�r�rr�rr�rX�c_floatr�r9r�r�r�rcr�r�r�r�r�r�r(r%r�r�5s&�������"'�%�8K�"L�"L�"L�F�L�L�L�L�"'�%���"G�"G�"G�K�G�G�G�G�&*�G�*�*�*�*��e�D�1�1�1�D�1�1�1�1�,1�E�$�,G�,G�,G�C�G�G�G�G�!�!�!�104�105�106�����I�I�I�I��9=�	i�i�i�i�i������r(r�c��eZdZdd�Zd	d�ZdS)107�
CustomSampler�108apply_func�2Callable[[llama_cpp.llama_token_data_array], None]c���|�_d�fd�}d	d�}tj��}tj|��|_|�_tjd��|_tjd��|_	tj109d��|_tjd��|_
tjd��|_tj���_t%j|���j_d�j_dS)110Nr2�llama_cpp.llama_sampler_p�cur_p�"llama_cpp.llama_token_data_array_pc�2����|��dSr)r�)r2r�r$s  �r%�
apply_wrapperz-CustomSampler.__init__.<locals>.apply_wrapperis���
�O�O�E�"�"�"�"�"r(c��dSrr�)r2s r%�free_wrapperz,CustomSampler.__init__.<locals>.free_wrapperos���Dr(r)r2r�r�r�)r2r�)r�r!�llama_sampler_i�llama_sampler_i_apply�apply�_apply_wrapper_ref�llama_sampler_i_namere�llama_sampler_i_acceptr��llama_sampler_i_resetrc�llama_sampler_i_clone�clone�llama_sampler_i_free�free�
llama_samplerr2rX�pointer�ifacer�)r$r�r�r��	sampler_is`    r%r4zCustomSampler.__init__ds����%���	#�	#�	#�	#�	#�	#�	�	�	�	��-�/�/�	�#�9�-�H�H�	��"/���"�7��:�:�	��$�;�A�>�>�	��#�9�!�<�<�	��#�9�!�<�<�	��"�7��:�:�	�� �.�0�0���#�^�I�6�6���������r(rAr�c�4�tj|j��Sr)rXr�r2r#s r%�get_samplerzCustomSampler.get_sampler�s���~�d�l�+�+�+r(N�r�r�)rAr�)r�r�r�r4r�r�r(r%r�r�cs<������ � � � �8,�,�,�,�,�,r(r�c���eZdZd�Zd�Zd�Zd�ZdJd�Zd�ZdKd111�Z	dLdMd�Z112dLdMd�ZdLdMd�ZdNd�Z
dOd�ZdPd�ZdQd�ZdRd�ZdSd �ZdTd%�ZdUd*�ZdVd/�ZdWd6�ZdXd9�ZdYd:�ZdZd=�Zd[d?�Zd\d]dD�Zd^dF�ZdG�ZdH�ZdIS)_�LlamaSamplerc����tj��}tj|���_g�_t���_�fd�}�j�|��dS)Nc�����j�Vt�j��D]\}}tj�j|��� tj�j��d�_dSdSr)r2r6r7r!r8�llama_sampler_free)r;r<r$s  �r%�free_samplerz+LlamaSampler.__init__.<locals>.free_sampler�si����|�'�$�T�%9�:�:�J�J�D�A�q��8���q�I�I�I�I��,�T�\�:�:�:�#�����(�'r()r!�"llama_sampler_chain_default_params�llama_sampler_chain_initr2r7r
r)r3)r$rr�s`  r%r4zLlamaSampler.__init__�so����=�?�?�� �9�&�A�A���@B���$�;�;���	$�	$�	$�	$�	$�	
��!�!�,�/�/�/�/�/r(c�8�|j���dSrr�r#s r%r:zLlamaSampler.close�r�r(c�.�|���dSrr>r#s r%r?zLlamaSampler.__del__�r@r(c�`�tj��}tj|j|��dSr)r!�llama_sampler_init_greedy�llama_sampler_chain_addr2�r$r2s  r%�113add_greedyzLlamaSampler.add_greedy�s+���5�7�7���)�$�,��@�@�@�@�@r(rrBc�b�tj|��}tj|j|��dSr)r!�llama_sampler_init_distr�r2)r$rr2s   r%�add_distzLlamaSampler.add_dist��-���3�D�9�9���)�$�,��@�@�@�@�@r(c�`�tj��}tj|j|��dSr)r!�llama_sampler_init_softmaxr�r2r�s  r%�add_softmaxzLlamaSampler.add_softmax�s+���6�8�8���)�$�,��@�@�@�@�@r(r*c�b�tj|��}tj|j|��dSr)r!�llama_sampler_init_top_kr�r2)r$r*r2s   r%�	add_top_kzLlamaSampler.add_top_k�s-���4�Q�7�7���)�$�,��@�@�@�@�@r(rr.rRr+c�d�tj||��}tj|j|��dSr)r!�llama_sampler_init_top_pr�r2�r$r.r+r2s    r%�	add_top_pzLlamaSampler.add_top_p��/���4�Q��A�A���)�$�,��@�@�@�@�@r(c�d�tj||��}tj|j|��dSr)r!�llama_sampler_init_min_pr�r2r�s    r%�	add_min_pzLlamaSampler.add_min_p�rr(c�d�tj||��}tj|j|��dSr)r!�llama_sampler_init_typicalr�r2r�s    r%�add_typicalzLlamaSampler.add_typical�s/���6�q�(�C�C���)�$�,��@�@�@�@�@r(r6c�b�tj|��}tj|j|��dSr)r!�llama_sampler_init_tempr�r2)r$r6r2s   r%�add_tempzLlamaSampler.add_temp�r�r(�t�delta�exponentc�f�tj|||��}tj|j|��dSr)r!�llama_sampler_init_temp_extr�r2)r$r114rrr2s     r%�add_temp_extzLlamaSampler.add_temp_ext�s1���7��5�(�K�K���)�$�,��@�@�@�@�@r(c�h�tj||||��}tj|j|��dSr)r!�llama_sampler_init_xtcr�r2)r$r.r115r+rr2s      r%�add_xtczLlamaSampler.add_xtc�s3���2�1�a��4�H�H���)�$�,��@�@�@�@�@r(r�c�b�tj|��}tj|j|��dSr)r!�llama_sampler_init_top_n_sigmar�r2)r$r�r2s   r%�add_top_n_sigmazLlamaSampler.add_top_n_sigma�s-���:�1�=�=���)�$�,��@�@�@�@�@r(rIr<r=r>c�j�tj|||||��}tj|j|��dSr)r!�llama_sampler_init_mirostatr�r2)r$rIrr<r=r>r2s       r%�add_mirostatzLlamaSampler.add_mirostat�s6���7���s�C�QR�S�S���)�$�,��@�@�@�@�@r(c�f�tj|||��}tj|j|��dSr)r!�llama_sampler_init_mirostat_v2r�r2)r$rr<r=r2s     r%�add_mirostat_v2zLlamaSampler.add_mirostat_v2�s1���:�4��c�J�J���)�$�,��@�@�@�@�@r(r rr9rc���tj|j|j�d��|j�d����}tj|j|��dSrm)r!�llama_sampler_init_grammarr1�_grammarr/�_rootr�r2)r$r r9r2s    r%�add_grammarzLlamaSampler.add_grammar�s\���6��K��)�0�0��9�9�7�=�;O�;O�PW�;X�;X�116�117��	�)�$�,��@�@�@�@�@r(�trigger_patterns�	List[str]�trigger_tokensr�c���tjt|��z��}t|��D]\}}|�d��||<�tjt|��z|�}tj|j|j	�d��|j118�d��|t|��|t|����}	tj|j|	��dSrm)
rX�c_char_pr��	enumerater/r!r��(llama_sampler_init_grammar_lazy_patternsr1rrr�r2)119r$r r9r!r#�pattern_ptrsr;�pattern�token_arrayr2s120          r%�add_grammar_lazy_patternsz&LlamaSampler.add_grammar_lazy_patterns�s�����#�.>�*?�*?�?�B�B��#�$4�5�5�	6�	6�J�A�w�%�n�n�W�5�5�L��O�O�!�,�s�>�/B�/B�B�^�T���D��K���#�#�G�,�,��M� � ��)�)��� �!�!������121�122��	�)�$�,��@�@�@�@�@r(r r!r"r#c�h�tj||||��}tj|j|��dSr)r!�llama_sampler_init_penaltiesr�r2)r$r r!r"r#r2s      r%�
add_penaltieszLlamaSampler.add_penalties�s?���8�����	123�124��	�)�$�,��@�@�@�@�@r(rM�dry_multiplier�dry_base�dry_allowed_length�dry_penalty_last_n�seq_breakersc�4�tjt|��z��}t|��D]\}	}125|126�d��||	<�tj|j||||||t|����}tj|j	|��dSrm)127rXr%r�r&r/r!�llama_sampler_init_dryr1r�r2)r$r rMr/r0r1r2r3�breaker_ptrsr;�breakerr2s            r%�add_dryzLlamaSampler.add_dry�s�����#�l�*;�*;�;�>�>��#�L�1�1�	6�	6�J�A�w�%�n�n�W�5�5�L��O�O��2��K�����������	128�	129��	�)�$�,��@�@�@�@�@r(r��Dict[int, float]c�N�tjt|��z��}t|�����D]"\}\}}|||_|||_�#tj|t|��|��}tj|j	|��dSr)130r!�llama_logit_biasr�r&�itemsrk�bias�llama_sampler_init_logit_biasr�r2)r$rIr��131bias_arrayr;rkr=r2s        r%�add_logit_biaszLlamaSampler.add_logit_biass��� �0�3�z�?�?�B�E�E�132� )�*�*:�*:�*<�*<� =� =�	&�	&��A�}��t�"'�J�q�M��!%�J�q�M����9���133�O�O��134�135��136	�)�$�,��@�@�@�@�@r(c�l�tj|j��}tj|j|��dSr)r!�llama_sampler_init_infillr1r�r2)r$r r2s   r%�137add_infillzLlamaSampler.add_infill+s/���5�e�k�B�B���)�$�,��@�@�@�@�@r(r�r�c���t|��}|���}tj|j|��|j�tj|j��dz138|f��dS)Nr)r�r�r!r�r2r7r��llama_sampler_chain_n)r$r��custom_samplerr2s    r%�139add_customzLlamaSampler.add_custom/sv��'�z�2�2�� �,�,�.�.���)�$�,��@�@�@���#�#�
�
,�T�\�
:�
:�Q�
>��O�	140�	141�	142�	143�	144r(rAc�4�tj|j��Sr)r!�llama_sampler_get_seedr2r#s r%�get_seedzLlamaSampler.get_seed:s���/���=�=�=r(r�r�r�r�c�B�tj|j|j|��Sr)r!�llama_sampler_sampler2r�)r$r�r�s   r%r�zLlamaSampler.sample=s���-�d�l�C�G�S�I�I�Ir(rkc�:�tj|j|��dSr)r!�llama_sampler_acceptr2ros  r%r�zLlamaSampler.accept@s���&�t�|�U�;�;�;�;�;r(c�8�tj|j��dSr)r!�llama_sampler_resetr2r#s r%rczLlamaSampler.resetCs���%�d�l�3�3�3�3�3r(c�$��|jrtd���tj|j��}t145�t146���|�_g�_t���_�fd�}�j�	|���S)Nz7Cannot clone LlamaSampler that contains custom samplersc�Z���j�"tj�j��d�_dSdSr)r2r!r�)�new_samplers�r%r�z(LlamaSampler.clone.<locals>.free_samplerRs6����"�.��,�[�-@�A�A�A�&*��#�#�#�/�.r()147r7rir!�llama_sampler_cloner2r��__new__r
r)r3)r$�cloned_samplerr�rSs   @r%r�zLlamaSampler.cloneFs������	a�%�&_�`�`�`�"�6�t�|�D�D��"�*�*�<�8�8��,���&(��#�"+�+�+���	+�	+�	+�	+�	+�148	��(�(��6�6�6��r(NrT)r*rB)r)r.rRr+rB)r6rR)r149rRrrRrrR)r.rRr150rRr+rBrrB)r�rR)151rIrBrrBr<rRr=rRr>rB)rrBr<rRr=rR)r rr9r)r rr9rr!r"r#r�)r rBr!rRr"rRr#rR)r rrMrBr/rRr0rRr1rBr2rBr3r")rIrBr�r9)r rr�r�)r�)r�r�r�rBrArB)rkrB)r�r�r�r4r:r?r�r�r�r�r�rrr	rrrrrr r+r.r8r@rCrGrJr�r�rcr�r�r(r%r�r��s�������0�0�0� !�!�!����A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�A�4
A�
A�
A�
A�A�A�A�A�6A�A�A�A�$A�A�A�A�	152�	153�	154�	155�>�>�>�>�J�J�J�J�J�<�<�<�<�4�4�4�����r(r�)%�156__future__rr*rX�typingrrrrrr	r157�dataclassesrr�158contextlibr
�numpyr{�numpy.typing�npt�llama_types�
llama_grammarr�_utilsr�llama_cpp.llama_cppr!rr�r�rsr�r�r�r�r�r�r(r%�<module>rbs���"�"�"�"�"�"�	�	�	�	�
�
�
�
�������������������)�(�(�(�(�(�(�(� � � � � � ���������������'�'�'�'�'�'�*�*�*�*�*�*�'�'�'�'�'�'�R6�R6�R6�R6�R6�R6�R6�R6�j}8�}8�}8�}8�}8�}8�}8�}8�@</�</�</�</�</�</�</�</�~,�,�,�,�,�,�,�,�:)�)�)��?�?�?�?�?�?�?���?�4�*�*�*�*�*�*�*���*�Z,�,�,�,�,�,�,�,�BT�T�T�T�T�T�T�T�T�Tr(