Team Ai
Datasetpublic

codekingpro/portable-devtools

sourceHugging Faceupdated 5mo agoView on Hugging Face
1likes14kdownloads
bm25.cpython-313.pyc130 linesDownload Raw Back to __pycache__
1�

2��j�0�
�P�%SSKrSSKJr SSKJr SSKJr SSKJrJ	r	J3r4 SSKrSSKr
SSKJr SSKJrJrJrJr SSKJrJr SS	KJrJr SS5KJr SSKJrJr /SQr \"S
SSSS\"S
S9\ Vs/sHoS3PM	 snSSS9	/r!\"\\#S'"SS\5r$"SS\5r%gs snf)�N)�defaultdict)�get_all_start_methods)�Path)�Any�Iterable�Type)�SnowballStemmer)�define_cache_dir�6iter_batch�get_all_punctuation�remove_non_alphanumeric)�ParallelWorkerPool�Worker)�SparseEmbedding�SparseTextEmbeddingBase)�SimpleTokenizer)�SparseModelDescription�ModelSource)�arabic�danish�dutch�english�finnish�french�german�greek�	hungarian�italian�	norwegian�7portuguese�romanian�russian�spanish�swedish�tamil�turkishzQdrant/bm25z6BM25 as sparse embeddings meant to be used with Qdrantz8apache-2.0g{�G�z�?)�hf�.txtTz	mock.file)	�model�9vocab_size�description�license�10size_in_GB�sources�additional_files�requires_idf�11model_file�supported_bm25_modelsc�0^�\rSrSrSrS&S\S\S-S\S\S\S	\S12\S\S\S-S
\	4U4Sjjjr13\S\\
4Sj5r\S\S	\S\\4Sj5rS'S\S\S\\\-S\S\S-S\S\S-S\\4SjjrS(S\\\-S\S\S-S
\	S\\414SjjrS\\S\\4SjrS\\S\\4SjrS\\\-S
\	S\4SjrS\\S\\\44Sjr\S\S\4S j5rS!\\\-S
\	S\\4S"jr\S\S#4S$j5rS%rU=r$))�Bm25�=a�Implements traditional BM25 in a form of sparse embeddings.15Uses a count of tokens in the document to evaluate the importance of the token.16 17WARNING: This model is expected to be used with `modifier="idf"` in the sparse vector index of Qdrant.18 19BM25 formula:20 21score(q, d) = SUM[ IDF(q_i) * (f(q_i, d) * (k + 1)) / (f(q_i, d) + k * (1 - b + b * (|d| / avg_len))) ],22 23where IDF is the inverse document frequency, computed on Qdrant's side24f(q_i, d) is the term frequency of the token q_i in the document d25k, b, avg_len are hyperparameters, described below.26 27Args:28    model_name (str): The name of the model to use.29    cache_dir (str, optional): The path to the cache directory.30        Can be set using the `FASTEMBED_CACHE_PATH` env variable.31        Defaults to `fastembed_cache` in the system's temp directory.32    k (float, optional): The k parameter in the BM25 formula. Defines the saturation of the term frequency.33        I.e. defines how fast the moment when additional terms stop to increase the score. Defaults to 1.2.34    b (float, optional): The b parameter in the BM25 formula. Defines the importance of the document length.35        Defaults to 0.75.36    avg_len (float, optional): The average length of the documents in the corpus. Defaults to 256.0.37    language (str): Specifies the language for the stemmer.38    disable_stemmer (bool): Disable the stemmer.39Raises:40    ValueError: If the model_name is not in the format <org>/<model> e.g. BAAI/bge-base-en.41N�42model_name�	cache_dir�k�b�avg_len�language�token_max_length�disable_stemmer�specific_model_path�kwargsc43�t>�[TU]"X40U44D6 U[;a[US35eX`lX0lX@lXPlURU5n[[U55UlX�lURUURURURS9UlXpl[#[%55UlX�lU(a[#5UlSUlOD[#UR/URUR55Ul[1U5Ul[2Ulg)Nz language is not supported)�local_files_onlyr>)�super�__init__�supported_languages�45ValueErrorr;r8r9r:�_get_model_description�strr46r7�_specific_model_path�download_model�_local_files_only�47_model_dirr<�setr�punctuationr=�	stopwords�stemmer�_load_stopwordsr	r�	tokenizer)
�selfr6r7r8r9r:r;r<r=r>r?�model_description�	__class__s
            ��[D:\code\apps\devtools\python\user_packages\Python313\site-packages\fastembed/sparse/bm25.pyrC�
Bm25.__init__[s	���	����9�&�9��.�.���z�)C�D�E�E�$�M������� �7�7�48�C���-�i�8�9���$7�!��-�-���N�N�!�3�3� $� 9� 9�	.�49���!1���2�4�5���.���'*�u�D�N��D�L� ��!5�!5�d�o�o�t�}�}�!U�V�D�N�*�8�4�D�L�(����returnc��[$)z�Lists the supported models.50 51Returns:52    list[SparseModelDescription]: A list of SparseModelDescription objects containing the model information.53)r2��clss rU�_list_supported_models�Bm25._list_supported_models�s54��%�$rW�	model_dirc���XS3-nUR5(d/$[US5nUR5R5sSSS5 $!,(df   g=f)Nr(�r)�exists�open�read�55splitlines)r[r^r;�stopwords_path�fs     rUrP�Bm25._load_stopwords�sO��"�z��%6�6���$�$�&�&��I�
�.�#�
&�!��6�6�8�&�&�(�'�
&�
&�s�A�56A!�	documents�57batch_size�parallelrAc#�# �Sn[U[5(aU/nSn[U[5(a[U5U:aSnUbU(a,[	X45Hn	URU	5Shv�N M gUS:Xa[R"5nS[5;aSOSn58UUURURURURURURUUS.59n[U=(d SUR!5U60S9nUR""[	X4540UD6Hn	U	Hn
U
v� M	 M gN�7f)	NFTr�61forkserver�spawn)62r6r7r8r9r:r;r<r=rAr>�)�num_workers�worker�start_method)�63isinstancerG�list�lenr�	raw_embed�os�	cpu_countrr8r9r:r;r<r=r�_get_worker_class�ordered_map)rRr6r7rhrirjrAr>�is_small�batchrq�params�pool�records              rU�_embed_documents�Bm25._embed_documents�s,������i��%�%�"��I��H��i��&�&��9�~�64�*�����x�#�I�:���>�>�%�0�0�0�;��1�}��<�<�>��+7�;P�;R�+R�<�X_�L�(�&��V�V��V�V��<�<� �M�M�$(�$9�$9�#'�#7�#7�$4�':��F�&�$�M���-�-�/�)��D�65�)�)�*�Y�*K�V�v�V��#�F� �L�$�W�/1�s�A/E�1E�2CEc66+�# �URUR[UR5UUUURUR67S9Shv�N gN7f)ac68Encode a list of documents into list of embeddings.69We use mean pooling with attention so that the model can handle variable-length inputs.70 71Args:72    documents: Iterator of documents or single document to embed73    batch_size: Batch size for encoding -- higher values will use more memory, but be faster74    parallel:75        If > 1, data-parallel encoding will be used, recommended for offline encoding of large datasets.76        If 0, use all available cores.77        If None, don't use data-parallel processing, use default onnxruntime threading instead.78 79Returns:80    List of embeddings, one per document81)r6r7rhrirjrAr>N)rr6rGr7rJrH)rRrhrirjr?s     rU�embed�82Bm25.embed�sR���,�(�(�����$�.�.�)��!��!�3�3� $� 9� 9�)�83�	84�	85�s�A86A�A�
A�tokensc�F�/nUH�nUR5nX0R;aM$X@R;aM5[U5UR:�aMPUR87(aUR88R
U5OUnU(dM�URU5 M� U$�N)�lowerrMrNrtr<rO�	stem_word�append)rRr��stemmed_tokens�token�lower_token�
stemmed_tokens      rU�_stem�89Bm25._stem�s���$&���E��+�+�-�K��(�(�(���n�n�,���5�z�D�1�1�1��CG�<�<�D�L�L�2�2�;�?�U`�M��}��%�%�m�4�� �rWc���/nUHpn[U5nURRU5nURU5nUR	U5nUR[R"U55 Mr U$r�)r
rQ�tokenizer��_term_frequencyr�r�	from_dict)rRrh�90embeddings�documentr�r��token_id2values       rUru�Bm25.raw_embedsp��-/�91�!�H�.�x�8�H��^�^�,�,�X�6�F�!�Z�Z��/�N�!�1�1�.�A�N����o�7�7��G�H�"��rW�textsc��Sn[U[5(aU/OUnUH7n[U5nURR	U5nU[U5-
nM9 U$)Nr)rrrGr
rQr�rt)rRr�r?�	token_num�textr�r�s       rU�token_count�Bm25.token_countsZ���	�%�e�S�1�1���u���D�.�t�4�H��^�^�,�,�X�6�F���V��$�I���rWc	�T�0n[[5nUHnX4==S-
ss'M [U5nUHpnURU5nX4nXpRS--X&'X&==XpRSUR92-93UR94U-UR-----ss'Mr U$)aCalculate the term frequency part of the BM25 formula.95 96(97    f(q_i, d) * (k + 1)98) / (99    f(q_i, d) + k * (1 - b + b * (|d| / avg_len))100)101 102Args:103    tokens (list[str]): The list of tokens in the document.104 105Returns:106    dict[int, float]: The token_id to term frequency mapping.107rn)r�intrt�compute_token_idr8r9r:)rRr��tf_map�counterr��doc_len�token_id�num_occurrencess        rUr��Bm25._term_frequencys���$&��)4�S�)9��#�M��"�a�'�"�$��f�+��$�M��,�,�]�;�H�%�4�O�.�&�&�1�*�=�F�����&�&��D�F�F�108�T�V�V�g�-����<�<�3�!�
��	%��
rWr�c�@�[[R"U55$r�)�abs�mmh3�hash)r[r�s  rUr��Bm25.compute_token_id6s���4�9�9�U�#�$�$rW�queryc	+�~^# �[U[5(aU/nUH�n[U5nTRR	U5nTRU5n[R"[[U4SjU555[RS9n[R"U5n[XgS9v� M� g7f)z�To emulate BM25 behaviour, we don't need to use weights in the query, and109it's enough to just hash the tokens and assign a weight of 1.0 to them.110c3�F># �UHnTRU5v� M g7fr�)r�)�.0r�rRs  �rU�	<genexpr>�#Bm25.query_embed.<locals>.<genexpr>Fs����R�>�%��.�.�u�5�5�>�s�!)�dtype)�indices�valuesN)
rrrGr
rQr�r��np�arrayrsrL�int32�	ones_liker)rRr�r?r�r�r��	token_idsr�s`       rU�query_embed�Bm25.query_embed:s������e�S�!�!��G�E��D�*�4�0�D��^�^�,�,�T�2�F�!�Z�Z��/�N�����S�R�>�R�R�S��h�h��I��\�\�)�,�F�!�)�C�C��s�B:B=�111Bm25Workerc��[$r�)r�rZs rUrx�Bm25._get_worker_classLs���rW)
rKrHr:r9r7r=r8r;rMrOrNr<rQ)Ng333333�?g�?gp@r�(FN)�NFN)r�N) �__name__�112__module__�__qualname__�__firstlineno__�__doc__rG�floatr��boolrrC�classmethodrsrr\rrPrrrr�r�rur��dictr�r�r�rrx�__static_attributes__�
__classcell__)rTs@rUr4r4=s�����@!%����!� "� %�*.�.)��.)���:�.)��	.)�113�.)��
.)��.)��.)��.)�!�4�Z�.)��.)�.)�`�%�t�,B�'C�%��%��)��)��)��S�	�)��)��#�!&�*.�/!��/!��/!���#��&�	/!�114�/!���*�
/!��/!�!�4�Z�/!�115�/�	"�/!�h�#�	116���#��&�117��118���*�	119�120�121�122�/�	"�
123�@�D��I��$�s�)��(���9��124�o�	����x��}�!4�������d�3�i��D��e��4D��<�%�S�%�S�%��%�D��x��}�!4�D��D��Q`�Ha�D�$��$�|�"4���rWr4c	��\rSrSrS\S\S\4Sjr\S\S\S\SS4Sj5rS\	\125\\4S\	\126\\\
44S	jr\S\S\S\S\4S127j5rSrg)
r�iQr6r7r?c�4�UR"X40UD6Ulgr�)�init_embeddingr))rRr6r7r?s    rUrC�Bm25Worker.__init__Rs���(�(��I�&�I��128rWrXc��U"SXS.UD6$�N)r6r7�r�)r[r6r7r?s    rU�start�Bm25Worker.startZs���H�j�H��H�HrW�itemsc#�b# �UH%up#URRU5nX$4v� M' g7fr�)r)ru)rRr��idxr{�onnx_outputs     rU�process�Bm25Worker.process^s0��� �J�C��*�*�.�.�u�5�K��"�"� �s�-/c��[SXS.UD6$r�)r4)r6r7r?s   rUr��Bm25Worker.init_embeddinges���I�z�I�&�I�IrW)r)N)r�r�r�r�rGrrCr�r�r�tupler�rsrr��staticmethodr4r�r�r�rWrUr�r�Qs���J��J��J��	J��I�s�I�s�I�c�I�l�I��I�#��e�C��H�o�.�#�	�%��T�/�2�2�3�	4�#��J�3�J�3�J�#�J�$�J��JrWr�)&rv�collectionsr�multiprocessingr�pathlibr�typingrrrr��numpyr��py_rust_stemmersr	�fastembed.common.utilsr129rrr
�fastembed.parallel_processorrr�&fastembed.sparse.sparse_embedding_baserr� fastembed.sparse.utils.tokenizerr�"fastembed.common.model_descriptionrrrDr2rs�__annotations__r4r�)�langs0rU�<module>r�s���	�#�1��&�&���,���D��=�R���,���L����}�-�4G�H�4G�D�F�$�-�4G�H���130�7��t�2�3��Q�"�Q�hJ��J��wIs�.B#
codekingpro/portable-devtools · Team Ai