resquared/knowledge-base
1
1import os2import re3import logging4import requests5import lxml6import gradio as gr7from bs4 import BeautifulSoup8import openai9import pinecone10from lxml import etree11import tiktoken12import time13import PyPDF214import numpy as np15import csv16import json17from uuid import uuid418import certifi19import urllib3.contrib.pyopenssl20import nltk21 22# Download necessary nltk data23nltk.download("punkt")24nltk.download("averaged_perceptron_tagger")25 26# Configure SSL certificate27os.environ["SSL_CERT_FILE"] = certifi.where()28urllib3.contrib.pyopenssl.inject_into_urllib3()29 30# Configure logging31logging.basicConfig(level=logging.DEBUG)32logging.getLogger("pinecone").setLevel(logging.WARNING)33 34# Set OpenAI API key and Pinecone API key (secrets)35openai.api_key = os.environ.get('OPENAI_API_KEY')36PINECONE_API_KEY = os.environ.get('PINECONE_API_KEY')37 38# Initialize Pinecone39pinecone.init(api_key=PINECONE_API_KEY, environment="us-east1-gcp")40index_name = 'openai-index'41if index_name not in pinecone.list_indexes():42 pinecone.create_index(index_name, dimension=1536, metric='cosine')43 44index = pinecone.Index(index_name)45 46def create_embeddings(text_chunks, model="text-embedding-ada-002", encoding_name="cl100k_base"):47 embeddings = []48 try:49 prepared_chunks = [chunk.replace("\n", " ") for chunk in text_chunks]50 response = openai.Embedding.create(input=prepared_chunks, model=model)51 if response and "data" in response:52 for data in response["data"]:53 embedding = data["embedding"]54 if len(embedding) == 1536: # Check if the embedding has the correct dimension55 embeddings.append(embedding)56 else:57 print(f"Invalid embedding dimension: {len(embedding)}")58 return embeddings59 except Exception as e:60 print(f"Error creating embeddings: {e}")61 return embeddings # Always return embeddings, even if it's empty62 63def store_embeddings_in_pinecone(ids, embeddings_list, metadata_list):64 if len(ids) != len(embeddings_list) or len(ids) != len(metadata_list):65 print("Error: IDs, embeddings, and metadata lists must have the same length.")66 return67 68 to_upsert = []69 70 for i in range(len(ids)):71 id = ids[i]72 embedding = embeddings_list[i]73 metadata = metadata_list[i]74 metadata["timestamp"] = time.time()75 to_upsert.append((id, embedding, metadata))76 77 if metadata['source'] == 'document':78 print(f"Storing document: {metadata['filename']} with id: {id}")79 elif metadata['source'] == 'chat':80 print(f"Storing chat: {metadata.get('input')} with id: {id}")81 else:82 print(f"Storing {metadata['source']} with id: {id}")83 84 try:85 index.upsert(vectors=to_upsert) # Modify this line86 except Exception as e:87 print(f"Error storing embeddings in Pinecone: {e}")88 89 90 91def fetch_webpage_content(url):92 response = requests.get(url)93 soup = BeautifulSoup(response.content, 'lxml')94 95 text_elements = soup.select('p, h1, h2, h3, h4, h5, h6')96 text = ' '.join([element.get_text() for element in text_elements])97 return text98 99def process_directory(directory_path):100 print(f"Processing directory: {directory_path}")101 for file in os.listdir(directory_path):102 if file.endswith(".txt"):103 with open(os.path.join(directory_path, file), 'r') as f:104 text = f.read()105 embeddings = create_embeddings([text])106 id = str(uuid4())107 metadata = {'source': 'document', 'timestamp': time.time()}108 store_embeddings_in_pinecone([id], embeddings, [metadata])109 110def num_tokens_from_string(string: str, encoding_name: str) -> int:111 """Returns the number of tokens in a text string."""112 enc = tiktoken.get_encoding(encoding_name)113 tokens = enc.encode(string)114 return len(tokens)115 116def chunk_document(text, encoding_name, max_tokens):117 chunks = []118 current_chunk = []119 current_token_count = 0120 121 for sentence in text.split('. '):122 sentence_tokens = num_tokens_from_string(sentence, encoding_name)123 if current_token_count + sentence_tokens <= max_tokens:124 current_chunk.append(sentence)125 current_token_count += sentence_tokens126 else:127 chunks.append('. '.join(current_chunk))128 current_chunk = [sentence]129 current_token_count = sentence_tokens130 131 if current_chunk:132 chunks.append('. '.join(current_chunk))133 134 return chunks135 136def index_documents():137 max_tokens_per_chunk = 2048138 encoding_name = "cl100k_base"139 model = "text-embedding-ada-002"140 for filename in os.listdir("docs"):141 if filename.endswith(".txt"):142 with open(os.path.join("docs", filename), 'r') as f:143 text = f.read()144 elif filename.endswith(".pdf"):145 with open(os.path.join("docs", filename), 'rb') as f:146 pdf = PyPDF2.PdfFileReader(f)147 text = " ".join(page.extract_text() for page in pdf.pages)148 elif filename.endswith(".csv"):149 with open(os.path.join("docs", filename), 'r') as csvfile:150 reader = csv.reader(csvfile)151 text = " ".join(" ".join(row) for row in reader)152 else:153 continue # Skip other file types154 155 chunks = chunk_document(text, encoding_name, max_tokens_per_chunk)156 for i, chunk in enumerate(chunks):157 embeddings = create_embeddings([chunk], model, encoding_name)158 if embeddings:159 id = str(uuid4())160 metadata = {'source': 'document', 'filename': f"{filename}_chunk_{i}", 'text': chunk, 'timestamp': time.time()}161 store_embeddings_in_pinecone([id], embeddings, [metadata])162 163def chatbot(input_text):164 url_pattern = re.compile(r'https?://[^\s]+')165 url_match = url_pattern.search(input_text)166 embeddings = []167 webpage_text = ""168 169 if url_match:170 specific_url = url_match.group()171 input_text = url_pattern.sub('', input_text)172 webpage_text = fetch_webpage_content(specific_url)173 chunks = chunk_document(webpage_text, "cl100k_base", 2048)174 for chunk in chunks:175 chunk_embeddings = create_embeddings([chunk])176 if chunk_embeddings:177 id = str(uuid4())178 metadata = {'source': 'website', 'url': specific_url, 'text': chunk, 'timestamp': time.time()}179 store_embeddings_in_pinecone([id], chunk_embeddings, [metadata])180 input_text = f"From the website: {webpage_text}\n\nQuestion: {input_text}"181 embeddings = create_embeddings([webpage_text]) # Create embeddings from the webpage text182 id = str(uuid4())183 metadata = {'source': 'website', 'url': specific_url, 'timestamp': time.time()}184 store_embeddings_in_pinecone([id], embeddings, [metadata])185 186 # Query Pinecone for similar embeddings187 if embeddings: # Only query Pinecone if embeddings were created188 results = index.query(queries=[embeddings[0]], top_k=5) # Query Pinecone with the webpage embeddings189 else:190 # Create embeddings from the input text if no URL was found191 embeddings = create_embeddings([input_text])192 if embeddings: # Only query Pinecone if embeddings were created193 results = index.query(queries=[embeddings[0]], top_k=5) # Query Pinecone with the input text embeddings194 else:195 results = {'results': [{'matches': []}]} # Create an empty results object196 197 print("Results:", results)198 results_data = results['results'][0]['matches']199 result_ids = [result['id'] for result in results_data]200 201 # Find the best matches based on the weighted score and token limit202 token_limit = 3500203 similarity_threshold = 0.6 # Customize this value as needed204 205 user_prompt_tokens = num_tokens_from_string(input_text, "cl100k_base")206 207 matches = []208 total_tokens = user_prompt_tokens209 for result_id, result_data in zip(result_ids, results_data):210 if result_data['score'] >= similarity_threshold:211 fetched_data = index.fetch([result_id])['vectors'][result_id]['metadata']212 match_text = ""213 if fetched_data['source'] == 'document':214 match_text = f"From the document {fetched_data['filename']}: {fetched_data.get('text', '')}"215 elif fetched_data['source'] == 'website':216 match_text = f"From the website {fetched_data['url']}: {fetched_data.get('text', '')}"217 elif fetched_data['source'] == 'chat':218 match_text = f"Previously, I mentioned: {fetched_data.get('response')}"219 220 tokens_in_match = num_tokens_from_string(match_text, "cl100k_base")221 if total_tokens + tokens_in_match <= token_limit:222 matches.append(match_text)223 total_tokens += tokens_in_match224 225 # Prepare the final input text to be sent to ChatGPT-3.5226 input_text = f"{' '.join(matches)}\n{input_text}"227 228 # Print the final input text229 print(f"Final input text: {input_text}")230 231 # Get the response from ChatGPT-3.5232 response = get_gpt_response(input_text)233 234 return response235 236 237 238 # Get the response from ChatGPT-3.5239def get_gpt_response(prompt):240 headers = {241 "Content-Type": "application/json",242 "Authorization": f"Bearer {openai.api_key}",243 }244 245 data = {246 "model": "gpt-3.5-turbo",247 "messages": [248 {"role": "system", "content": "You are a customer success manager for Resquared that answers questions based on the knowledge base and adapts our pre-made templates based on user requests."},249 {"role": "user", "content": "generate a response and be sure to analyze the data from our knowledge base you are provided"},250 {251 "role": "user",252 "content": f"I am a client with a request based on the Resquared knowledge base. Resquared's product has data on every local business and tools for contacting them. Model your answer, in fact tone and style, on the following information from the Resquared knowledge base. If i ask you for a template, you should generate a concise and informal template that closely resembles the structure, length, tone, and style of the example templates I provided which should never be more than 90 words.: {prompt}"253 }254 ],255 "temperature": 0.5256 }257 258 try:259 response = requests.post(260 "https://api.openai.com/v1/chat/completions",261 headers=headers,262 data=json.dumps(data)263 )264 response.raise_for_status()265 except requests.exceptions.HTTPError as errh:266 return f"HTTP Error: {errh}"267 except requests.exceptions.ConnectionError as errc:268 return f"Error Connecting: {errc}"269 except requests.exceptions.Timeout as errt:270 return f"Timeout Error: {errt}"271 except requests.exceptions.RequestException as err:272 return f"Something went wrong: {err}"273 274 response_json = response.json()275 276 if "choices" in response_json and response_json["choices"]:277 return response_json["choices"][0]["message"]["content"].strip()278 else:279 return "Sorry, I couldn't generate a response."280 281 282# Launch Gradio interface283if __name__ == "__main__":284 index_documents()285 iface = gr.Interface(fn=chatbot,286 inputs=gr.inputs.Textbox(lines=7, label="Enter your text"),287 outputs=gr.outputs.Textbox(),288 title="Resquared Knowledge Base Bot")289 iface.launch()290