chriscob21/SupportChatBot_forWebsites

1from langchain.text_splitter import RecursiveCharacterTextSplitter2from langchain.vectorstores import Pinecone3from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings4import pinecone5import asyncio6from langchain.document_loaders.sitemap import SitemapLoader7 8 9#Function to fetch data from website10#https://python.langchain.com/docs/modules/data_connection/document_loaders/integrations/sitemap11def get_website_data(sitemap_url):12 13    loop = asyncio.new_event_loop()14    asyncio.set_event_loop(loop)15    loader = SitemapLoader(16    sitemap_url17    )18 19    docs = loader.load()20 21    return docs22 23#Function to split data into smaller chunks24def split_data(docs):25 26    text_splitter = RecursiveCharacterTextSplitter(27    chunk_size = 1000,28    chunk_overlap  = 200,29    length_function = len,30    )31 32    docs_chunks = text_splitter.split_documents(docs)33    return docs_chunks34 35#Function to create embeddings instance36def create_embeddings():37 38    embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")39    return embeddings40 41#Function to push data to Pinecone42def push_to_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings,docs):43 44    pinecone.init(45    api_key=pinecone_apikey,46    environment=pinecone_environment47    )48 49    index_name = pinecone_index_name50    index = Pinecone.from_documents(docs, embeddings, index_name=index_name)51    return index52 53#Function to pull index data from Pinecone54def pull_from_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings):55 56    pinecone.init(57    api_key=pinecone_apikey,58    environment=pinecone_environment59    )60 61    index_name = pinecone_index_name62 63    index = Pinecone.from_existing_index(index_name, embeddings)64    return index65 66#This function will help us in fetching the top relevent documents from our vector store - Pinecone Index67def get_similar_docs(index,query,k=2):68 69    similar_docs = index.similarity_search(query, k=k)70    return similar_docs71 72 73