Team Ai
Datasetpublic

codekingpro/portable-devtools

sourceHugging Faceupdated 5mo agoView on Hugging Face
1likes14kdownloads
minicoil.cpython-313.pyc137 linesDownload Raw Back to __pycache__
1�

2��j�:����%SSKJr SSKJrJrJrJr SSKrSSK	J3r4 SSKJr SSK
Jr SSKJrJr SSKJr SS	KJr SS5KJr SSKJr SSKJrJr SS
KJr SSKJ r J!r! SSK"J#r#J$r$ SSK%J&r&J'r' Sr(Sr)Sr*\"SSSSS\"SS9S\*\(\)/SS9	/r+\,\\-S'SS0r.\.R_5VVs0sHupURa5U_M snnr1S\2S \24S!jr3"S"S#\\&\5r4"S$S%\'\5r5gs snnf)&�)�Path)�Any�Sequence�Iterable�TypeN)�NDArray)�SnowballStemmer)�	Tokenizer)�SparseModelDescription�ModelSource)�OnnxOutputContext)�OnnxProvider)�Device)�define_cache_dir)�SparseEmbedding�SparseTextEmbeddingBase)�Encoder)�SparseVectorConverter�
WordEmbedding)�
VocabResolver�VocabTokenizer)�
OnnxTextModel�TextEmbeddingWorkerzminicoil.triplet.model.npyzminicoil.triplet.model.vocabz
stopwords.txtzQdrant/minicoil-v1i�Jz�Sparse embedding model, that resolves semantic meaning of the words, while keeping exact keyword match behavior. Based on jinaai/jina-embeddings-v2-small-en-tokensz6apache-2.0g7ףp=8�?)�hfzonnx/model.onnxT)	�model�9vocab_size�description�license�10size_in_GB�sources�11model_file�additional_files�requires_idf�supported_minicoil_models�english�12model_name�returnc�0�[UR5$�N)�MODEL_TO_LANGUAGE�lower)r&s �_D:\code\apps\devtools\python\user_packages\Python313\site-packages\fastembed/sparse/minicoil.py�get_language_by_model_namer-:s���Z�-�-�/�0�0�c��^�\rSrSrSrSSSSSS\RSSSS4S\S	\S-S13\S-S\	\14S-S\S
\S\S\\-S\
\S-S\S\S-S\S-S\4U4SjjjrS)SjrS*S\\\-S\S\S\4SjjrS+S\\\-S\S\S-S\S\\415SjjrS\\\-S\S\\4Sjr\S \S\
\4S!j5r\S\
\4S"j5rS,S#\S$\S\S\\4S%jjr\S\S&4S'j5rS(rU=r $)-�MiniCOIL�>a�16    MiniCOIL is a sparse embedding model, that resolves semantic meaning of the words,17    while keeping exact keyword match behavior.18 19    Each vocabulary token is converted into 4d component of a sparse vector, which is then weighted by the token frequency in the corpus.20    If the token is not found in the corpus, it is treated exactly like in BM25.21`22    The model is based on `jinaai/jina-embeddings-v2-small-en-tokens`23Ng333333�?g�?g�b@Fr&�	cache_dir�threads�	providers�k�b�avg_len�cuda�24device_ids�	lazy_load�	device_id�specific_model_path�kwargsc
�>�[TU]"XU40U
D6 X@lX�lX�lX�lX�lURU
5UlXPl	X`l25XplSUl0Ul
[5Ul[5Ul[5UlSUlSUlSUlSUlUR-U5Ul[1[3U55UlX�lUR9UR.UR4UR:UR6S9UlUR(dUR?5 gg)ay26Args:27    model_name (str): The name of the model to use.28    cache_dir (str, optional): The path to the cache directory.29                               Can be set using the `FASTEMBED_CACHE_PATH` env variable.30                               Defaults to `fastembed_cache` in the system's temp directory.31    threads (int, optional): The number of threads single onnxruntime session can use. Defaults to None.32    providers (Optional[Sequence[OnnxProvider]], optional): The providers to use for onnxruntime.33    k (float, optional): The k parameter in the BM25 formula. Defines the saturation of the term frequency.34        I.e. defines how fast the moment when additional terms stop to increase the score. Defaults to 1.2.35    b (float, optional): The b parameter in the BM25 formula. Defines the importance of the document length.36        Defaults to 0.75.37    avg_len (float, optional): The average length of the documents in the corpus. Defaults to 150.0.38    cuda (Union[bool, Device], optional): Whether to use cuda for inference. Mutually exclusive with `providers`39        Defaults to Device.AUTO.40    device_ids (Optional[list[int]], optional): The list of device ids to use for data parallel processing in41        workers. Should be used with `cuda` equals to `True`, `Device.AUTO` or `Device.CUDA`, mutually exclusive42        with `providers`. Defaults to None.43    lazy_load (bool, optional): Whether to load the model during class initialization or on demand.44        Should be set to True when using multiple-gpu and parallel encoding. Defaults to False.45    device_id (Optional[int], optional): The device id to use for loading the model in the worker process.46    specific_model_path (Optional[str], optional): The specific path to the onnx model dir if it should be imported from somewhere else47 48Raises:49    ValueError: If the model_name is not in the format <org>/<model> e.g. BAAI/bge-base-en.50N)�local_files_onlyr<) �super�__init__r4r:r9r8r;�_select_exposed_session_options�_extra_session_optionsr5r6r7�	tokenizer�invert_vocab�set�special_tokens�special_tokens_ids�	stopwords�vocab_resolver�encoder�51output_dim�sparse_vector_converter�_get_model_description�model_description�strrr2�_specific_model_path�download_model�_local_files_only�52_model_dir�load_onnx_model)�selfr&r2r3r4r5r6r7r8r9r:r;r<r=�	__class__s              �r,rA�MiniCOIL.__init__Is���V	�����B�6�B�"��"��$���	�"��&*�&J�&J�6�&R��#�������,0���,.���(+����,/�E���#&�5���48���'+���&*���EI��$�!%�!<�!<�Z�!H����-�i�8�9���$7�!��-�-��"�"��N�N�!�3�3� $� 9� 9�	.�53����~�~�� � �"�r.r'c54��URURURRURUR55URURURS9 URceURR5R5HupXRU'M [URR55Ul[URR#55Ul[UR'UR55Ul[+[-UR.55n[1[3UR5UR(US9UlUR4R7[9UR[:-55 [<R>"[9UR[@-5SS9n[CU5Ul"URDRFUl#[IUR(UURJURLURNS9Ul(g)N)�	model_dirr!r3r4r8r;�extra_session_options)rDrI�stemmer�r)�	mmap_mode)rIr\r5r6r7))�_load_onnx_modelrTrOr!r3r4r8r;rCrD�	get_vocab�itemsrErF�special_token_to_id�keysrG�valuesrH�_load_stopwordsrIr	r-r&rrrJ�load_json_vocabrP�MINICOIL_VOCAB_FILE�np�load�MINICOIL_MODEL_FILErrKrLrr5r6r7rM)rV�token�idxr\�weightss     r,rU�MiniCOIL.load_onnx_model�s�������o�o��-�-�8�8��L�L��n�n�����n�n�"&�"=�"=�	�	56��~�~�)�)�)��.�.�2�2�4�:�:�<�J�E�%*���c�"�=�!�$�":�":�"?�"?�"A�B���"%�d�&>�&>�&E�&E�&G�"H����T�1�1�$�/�/�B�C���!�"<�T�_�_�"M�N��+�$�T�^�^�4��n�n��57���58	
���+�+�C����BU�0U�,V�W��'�'�#�d�o�o�0C�C�D�PS�T���w�'����,�,�1�1���'<��n�n���f�f��f�f��L�L�(59��$r.�texts�60batch_sizec�,�UR"U4SU0UD6$)Nrp)�_token_count)rVrorpr=s    r,�token_count�MiniCOIL.token_count�s��� � ��H�:�H��H�Hr.�	documents�parallelc+�R# �UR"SUR[UR5UUUURUR61URURURURSURURURS.UD6Shv�N gN7f)ac62Encode a list of documents into list of embeddings.63We use mean pooling with attention so that the model can handle variable-length inputs.64 65Args:66    documents: Iterator of documents or single document to embed67    batch_size: Batch size for encoding -- higher values will use more memory, but be faster68    parallel:69        If > 1, data-parallel encoding will be used, recommended for offline encoding of large datasets.70        If 0, use all available cores.71        If None, don't use data-parallel processing, use default onnxruntime threading instead.72 73Returns:74    List of embeddings, one per document75F)r&r2rurprvr4r8r9r5r6r7�is_queryr?r<r[N�)
�_embed_documentsr&rPr2r4r8r9r5r6r7rSrQrC)rVrurprvr=s     r,�embed�MiniCOIL.embed�s����,�(�(�76�����$�.�.�)��!���n�n��������f�f��f�f��L�L��!�3�3� $� 9� 9�"&�"=�"=�77� �!78�	79�	80�s�BB'�B%� B'�queryc+�8# �UR"SUR[UR5UURUR81URURURURSURURS.UD6Shv�N gN7f)z382Encode a list of queries into list of embeddings.83T)r&r2rur4r8r9r5r6r7rxr?r<Nry)rzr&rPr2r4r8r9r5r6r7rSrQ)rVr}r=s   r,�query_embed�MiniCOIL.query_embed�s�����(�(�84�����$�.�.�)���n�n��������f�f��f�f��L�L��!�3�3� $� 9� 9�85��86�	87�	88�s�BB�B�BrZc���U[-nUR5(d/$[US5nUR5R	5sSSS5 $!,(df   g=f)Nr])�STOPWORDS_FILE�exists�open�read�89splitlines)�clsrZ�stopwords_path�fs    r,re�MiniCOIL._load_stopwordssJ��"�^�3���$�$�&�&��I�
�.�#�
&�!��6�6�8�&�&�(�'�
&�
&�s�A�90A#c��[$)z�Lists the supported models.91 92Returns:93    list[SparseModelDescription]: A list of SparseModelDescription objects containing the model information.94)r$�r�s r,�_list_supported_models�MiniCOIL._list_supported_modelss95��)�(r.�outputrxc96+�# �URc[S5eURceURceURceUR97nURceURnURR5nURRn[URS5GH�nXHXXS:H4n	URX�US:H4n98URRU995up�p�[R"USS9n[R"U	SS9nURSURS:XdeURRUU5unnUSS2S4R5n0nUVs/sHnURR!U5PM  nn[#UUUR55H7unnnUS:XaM[%UUU['UU5['U5US9UU'M9 U
R)5H!unn[%UU/['U5SS/S9UU'M# U(dURR+UXgS9v� GM�URR-UXgS9v� GM� gs snf7f)Nz7input_ids must be provided for document post-processingr�)�axis)�word�forms�count�word_id�	embedding�����)r�embedding_size)�	input_ids�100ValueErrorrJrKrM�model_output�attention_maskrrL�range�shape�resolve_tokensrh�expand_dims�forward�tolist�lookup_word�zipr�intra�embedding_to_vector�embedding_to_vector_query)rVr�rxr=�101embeddings�masksrr��i�token_embeddings�	token_ids�word_ids_array�counts�oovr��word_ids_array_expanded�token_embeddings_array�ids_mapping�minicoil_embeddings�	words_ids�sentence_resultr��wordsr��emb�oov_wordr�s                           r,�_post_process_onnx_output�"MiniCOIL._post_process_onnx_outputs�������#��V�W�W��"�"�.�.�.��|�|�'�'�'��+�+�7�7�7��(�(�102��$�$�0�0�0��%�%���(�(�3�3�5�103����0�0���z�'�'��*�+�A�)�U�X��]�*:�;��,2�+;�+;�A�Q�x�1�}�<L�+M�I�15�1D�1D�1S�1S�T]�1^�.�N�C�:<����]^�9_�#�;=�.�.�IY�`a�:b�"�*�0�0��3�7M�7S�7S�TU�7V�V�V�V�04�|�|�/C�/C�'�)?�0�,�K�,�104$/�q�!�t�#4�#;�#;�#=�I�8:�O�MV�W�Y�'�T�(�(�4�4�W�=�Y�E�W�&)�%��<O�<V�<V�<X�&Y�"��g�s��a�<��(5����+��f�W�o�.���L�!�)���%�	'Z�$'�9�9�;���%�-:�!�(��3�u�:�r�^_�]`�-���)�$/���2�2�F�F�#�105�G����2�2�L�L�#�106�M���w,��8X�s�FJ�%I<�'CJ�MiniCoilTextEmbeddingWorkerc��[$r))r�r�s r,�_get_worker_class�MiniCOIL._get_worker_classhs��*�*r.)rCrTrQr7r6r2r8r;r9rKrEr5r:rOrLr4rMrGrHrIrDrJ)r'N)i)�N)F)!�__name__�107__module__�__qualname__�__firstlineno__�__doc__r�AUTOrPr�rr�float�bool�listrrArUrrsrr{r�classmethodrrerr�r
r�rr��__static_attributes__�
__classcell__)rWs@r,r0r0>sF����!%�"�37����$�k�k�'+�� $�*.�M#��M#���:�M#��t��	M#�108�L�)�D�0�M#��
M#��M#��M#��V�m�M#���I��$�M#��M#���:�M#�!�4�Z�M#��M#�M#�^&109�R=A�I��8�C�=�(�I�69�I�LO�I�	�I��#�	'110���#��&�'111��'112���*�	'113�114�'115�116�/�	"�
'117�R118��x��}�!4�119��120��Q`�Ha�121�(�)��)��c��)��)��)�t�,B�'C�)��)�;@�R�'�R�37�R�KN�R�	�/�	"�R�h�+�$�'D�"E�+��+r.r0c�.�\rSrSrS\S\S\S\4SjrSrg)	r�imr&r2r=r'c� �[SUUSS.UD6$)Nr�)r&r2r3ry)r0)rVr&r2r=s    r,�init_embedding�*MiniCoilTextEmbeddingWorker.init_embeddingns'���122�!���123��	124�	125r.ryN)	r�r�r�r�rPrr0r�r�ryr.r,r�r�ms$��126��127��128��129�PX�130r.r�)6�pathlibr�typingrrrr�numpyrh�numpy.typingr�py_rust_stemmersr	�131tokenizersr132�"fastembed.common.model_descriptionrr�fastembed.common.onnx_modelr
�fastembed.commonr�fastembed.common.typesr�fastembed.common.utilsr�&fastembed.sparse.sparse_embedding_baserr�'fastembed.sparse.utils.minicoil_encoderr�/fastembed.sparse.utils.sparse_vectors_converterrr�%fastembed.sparse.utils.vocab_resolverrr�fastembed.text.onnx_text_modelrrrjrgr�r$r��__annotations__�_MODEL_TO_LANGUAGErar+r*rPr-r0r�)r&�languages00r,�<module>r�s)���0�0�� �,� �R�9�)�)�3��<�`�O�M�3��4�� ���"��=����3�4�$����133�134��;��4� 6�7��*�)���>P�=U�=U�=W��=W�%9�Z�J����� �=W���1351�3�1�3�1�l+�&�
�o�(F�l+�^	136�"5�o�"F�137��q	s�+C3
codekingpro/portable-devtools · Team Ai