Somnath3570/PDF_based_knowledge_management_system

1from langchain_community.document_loaders import PyPDFLoader, DirectoryLoader
2from langchain.text_splitter import RecursiveCharacterTextSplitter
3from langchain_huggingface import HuggingFaceEmbeddings
4from langchain_community.vectorstores import FAISS
5
6## Uncomment the following files if you're not using pipenv as your virtual environment manager
7from dotenv import load_dotenv, find_dotenv
8load_dotenv(find_dotenv())
9
10
11# Step 1: Load raw PDF(s)
12DATA_PATH="data/"
13def load_pdf_files(data):
14    loader = DirectoryLoader(data,
15                             glob='*.pdf',
16                             loader_cls=PyPDFLoader)
17    
18    documents=loader.load()
19    return documents
20
21documents=load_pdf_files(data=DATA_PATH)
22#print("Length of PDF pages: ", len(documents))
23
24
25# Step 2: Create Chunks
26def create_chunks(extracted_data):
27    text_splitter=RecursiveCharacterTextSplitter(chunk_size=500,
28                                                 chunk_overlap=50)
29    text_chunks=text_splitter.split_documents(extracted_data)
30    return text_chunks
31
32text_chunks=create_chunks(extracted_data=documents)
33#print("Length of Text Chunks: ", len(text_chunks))
34
35# Step 3: Create Vector Embeddings 
36
37def get_embedding_model():
38    embedding_model=HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
39    return embedding_model
40
41embedding_model=get_embedding_model()
42
43# Step 4: Store embeddings in FAISS
44DB_FAISS_PATH="vectorstore/db_faiss"
45db=FAISS.from_documents(text_chunks, embedding_model)
46db.save_local(DB_FAISS_PATH)
47