Team Ai
Apppublic

MachineLearningReply/docwhiz

sourceHugging Facemitupdated 3y agoView on Hugging Face
0likes
rootutils__pycache__haystack.cpython-39.pyc
haystack.cpython-39.pyc33 linesDownload Raw Back to __pycache__
1a

2��f��@s ddlZddlmZmZddlmZddlmZddl	m3Z4ddl	mZmZm
Z
ddlmZmZmZmZddlmZejd	d5�dd��Zejd	d6�ed
�dd��Zejd	d7�e8d�dd��Zejd	d9�dd��Zejd	d10�e11eed�dd��Zejd	d12�e13ed�dd��Zdd�Zd dd�ZdS)!�N)�document_store_configs�
model_configs)�Pipeline)�Answer)�BaseDocumentStore)�InMemoryDocumentStore�OpenSearchDocumentStore�WeaviateDocumentStore)�EmbeddingRetriever�14FARMReader�15PromptNode�PreProcessor)�MilvusDocumentStoreF)�show_spinnercCs td�tddddddd�}|S)Nzinitializing preprocessor nodeT�word�d)Zclean_empty_linesZclean_whitespaceZclean_header_footerZsplit_byZsplit_lengthZsplit_respect_sentence_boundary)�printr
)�	processor�r�;/Users/a.madhigirigopinath/search_mlReply/utils/haystack.py�start_preprocessor_node
s�r)�typec	Cs�td�|dkrtddd�}n�|dkrZttdtdtd	td16tdtdtd
d�}nN|dkr�ttdtdtdtdd�}n$|dkr�ttdtdtddd�}|S)Nzinitializing document store�inmemoryTi�)Zuse_bm25�
embedding_dim�17opensearch�OPENSEARCH_SCHEME�OPENSEARCH_USERNAME�OPENSEARCH_PASSWORD�OPENSEARCH_HOST�OPENSEARCH_PORT�OPENSEARCH_INDEX�OPENSEARCH_EMBEDDING_DIM)�scheme�username�password�host�port�indexr�weaviate�
WEAVIATE_HOST�
WEAVIATE_PORT�WEAVIATE_INDEX�WEAVIATE_EMBEDDING_DIM)r%r&r'r�milvus�18MILVUS_URI�MILVUS_INDEX�MILVUS_EMBEDDING_DIM)�urir'rZreturn_embedding)rrrrr	r)r�document_storerrr�start_document_stores4
���r3)�_document_storecCstd�t|tddd�}|S)Nzinitializing retriever�EMBEDDING_MODEL�)r2Zembedding_modelZtop_k)rr19r)r4�	retrieverrrr�start_retrieverFs�r8cCstd�ttdd�}|S)Nzinitializing reader�EXTRACTIVE_MODEL)�model_name_or_path)rrr)�readerrrr�start_readerRsr<)r4�20_retriever�_readercCs6td�t�}|j|ddgd�|j|ddgd�|S)Nzinitializing pipeline�	Retriever�Query��	component�name�inputs�Reader)rr�add_node)r4r=r>�piperrr�start_haystack_extractive[s21rH)r4r=cCsBtdtd|dd�}t�}|j|ddgd�|j|ddgd�|S)	Nzdeepset/question-answering�GENERATIVE_MODELi�)Zdefault_prompt_templater:Zapi_key�22max_lengthr?r@rAr)rrrrF)r4r=�23openai_keyZprompt_noderGrrr�start_haystack_ragcs�rLcCsi}|j||d�}|S)N)�params)�run)Z	_pipeline�questionrM�resultsrrr�queryqsrQ�cCs,|dkrt|||�S|dkr(t|||�SdS)N�24extractive�rag)rHrL)�taskr2r7r;rKrrr�initialize_pipelinevsrV)rR)�	streamlit�st�utils.configrr�haystackrZhaystack.schemarZhaystack.document_storesrrrr	Zhaystack.nodesr25rrr
Zmilvus_haystackr�cache_resourcer�strr3r8r<rHrLrQrVrrrr�<module>s*262728'2930313233