Team Ai
Apppublic

MachineLearningReply/docwhiz

sourceHugging Facemitupdated 3y agoView on Hugging Face
0likes
rootutils__pycache__haystack.cpython-310.pyc
haystack.cpython-310.pyc35 linesDownload Raw Back to __pycache__
1o

2��f��@s&ddlZddlmZmZddlmZddlmZddl	m3Z4ddl	mZmZm
Z
ddlmZmZmZmZddlmZejd	d5�dd��Zejd	d6�d
efdd��Zejd	d7�de8fdd��Zejd	d9�dd��Zejd	d10�de11dedefdd��Zejd	d12�de13defdd��Zdd�Zd dd�ZdS)!�N)�document_store_configs�
model_configs)�Pipeline)�Answer)�BaseDocumentStore)�InMemoryDocumentStore�OpenSearchDocumentStore�WeaviateDocumentStore)�EmbeddingRetriever�14FARMReader�15PromptNode�PreProcessor)�MilvusDocumentStoreF)�show_spinnercCs td�tddddddd�}|S)Nzinitializing preprocessor nodeT�word�d)Zclean_empty_linesZclean_whitespaceZclean_header_footerZsplit_byZsplit_lengthZsplit_respect_sentence_boundary)�printr
)�	processor�r�;/Users/a.madhigirigopinath/search_mlReply/utils/haystack.py�start_preprocessor_node
s�r�typec	Cs�td�|dkrtddd�}	|S|dkr0ttdtdtd	td16tdtdtd
d�}|S|dkrFttdtdtdtdd�}|S|dkrXttdtdtddd�}|S)Nzinitializing document store�inmemoryTi�)Zuse_bm25�
embedding_dim�17opensearch�OPENSEARCH_SCHEME�OPENSEARCH_USERNAME�OPENSEARCH_PASSWORD�OPENSEARCH_HOST�OPENSEARCH_PORT�OPENSEARCH_INDEX�OPENSEARCH_EMBEDDING_DIM)�scheme�username�password�host�port�indexr�weaviate�
WEAVIATE_HOST�
WEAVIATE_PORT�WEAVIATE_INDEX�WEAVIATE_EMBEDDING_DIM)r%r&r'r�milvus�18MILVUS_URI�MILVUS_INDEX�MILVUS_EMBEDDING_DIM)�urir'rZreturn_embedding)rrrrr	r)r�document_storerrr�start_document_stores:����	��r3�_document_storecCstd�t|tddd�}|S)Nzinitializing retriever�EMBEDDING_MODEL�)r2Zembedding_modelZtop_k)rr19r)r4�	retrieverrrr�start_retrieverFs�r8cCstd�ttdd�}|S)Nzinitializing reader�EXTRACTIVE_MODEL)�model_name_or_path)rrr)�readerrrr�start_readerRsr<�20_retriever�_readercCs6td�t�}|j|ddgd�|j|ddgd�|S)Nzinitializing pipeline�	Retriever�Query��	component�name�inputs�Reader)rr�add_node)r4r=r>�piperrr�start_haystack_extractive[s21rHcCsBtdtd|dd�}t�}|j|ddgd�|j|ddgd�|S)	Nzdeepset/question-answering�GENERATIVE_MODELi�)Zdefault_prompt_templater:Zapi_key�22max_lengthr?r@rAr)rrrrF)r4r=�23openai_keyZprompt_noderGrrr�start_haystack_ragcs�rLcCsi}|j||d�}|S)N)�params)�run)Z	_pipeline�questionrM�resultsrrr�queryqsrQ�cCs,|dkr24t|||�S|dkrt|||�SdS)N�25extractive�rag)rHrL)�taskr2r7r;rKrrr�initialize_pipelinevs26�rV)rR)�	streamlit�st�utils.configrr�haystackrZhaystack.schemarZhaystack.document_storesrrrr	Zhaystack.nodesr27rrr
Zmilvus_haystackr�cache_resourcer�strr3r8r<rHrLrQrVrrrr�<module>s,28293031'32333435