666lcz/knowledge_base

1{2 "cells": [3  {4   "cell_type": "code",5   "execution_count": 1,6   "id": "7b04fff4",7   "metadata": {},8   "outputs": [],9   "source": [10    "from langchain.document_loaders import WebBaseLoader\n",11    "\n",12    "loader = WebBaseLoader(\"https://lilianweng.github.io/posts/2023-06-23-agent/\")\n",13    "data = loader.load()"14   ]15  },16  {17   "cell_type": "code",18   "execution_count": 2,19   "id": "6fb0f09e",20   "metadata": {},21   "outputs": [],22   "source": [23    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",24    "\n",25    "text_splitter = RecursiveCharacterTextSplitter(chunk_size = 500, chunk_overlap = 0)\n",26    "all_splits = text_splitter.split_documents(data)"27   ]28  },29  {30   "cell_type": "code",31   "execution_count": 3,32   "id": "e1595d97",33   "metadata": {},34   "outputs": [35    {36     "data": {37      "text/plain": [38       "True"39      ]40     },41     "execution_count": 3,42     "metadata": {},43     "output_type": "execute_result"44    }45   ],46   "source": [47    "# Set env var OPENAI_API_KEY or load from a .env file\n",48    "import dotenv\n",49    "\n",50    "dotenv.load_dotenv()"51   ]52  },53  {54   "cell_type": "code",55   "execution_count": 5,56   "id": "0f12e3f0",57   "metadata": {},58   "outputs": [],59   "source": [60    "from langchain.embeddings import OpenAIEmbeddings\n",61    "from langchain.vectorstores import Chroma\n",62    "\n",63    "vectorstore = Chroma.from_documents(documents=all_splits, embedding=OpenAIEmbeddings())"64   ]65  },66  {67   "cell_type": "code",68   "execution_count": 8,69   "id": "caaefacc",70   "metadata": {},71   "outputs": [72    {73     "data": {74      "text/plain": [75       "'Task decomposition can be done (1) by LLM with simple prompting like \"Steps for XYZ.\\\\n1.\", \"What are the subgoals for achieving XYZ?\", (2) by using task-specific instructions; e.g. \"Write a story outline.\" for writing a novel, or (3) with human inputs.'"76      ]77     },78     "execution_count": 8,79     "metadata": {},80     "output_type": "execute_result"81    }82   ],83   "source": [84    "# query = \"What did the president say about Ketanji Brown Jackson\"\n",85    "# docs = db.similarity_search(query)\n",86    "# print(docs[0].page_content)\n",87    "question = \"What are the approaches to Task Decomposition?\"\n",88    "docs = vectorstore.similarity_search(question)\n",89    "docs[0].page_content"90   ]91  },92  {93   "cell_type": "code",94   "execution_count": 9,95   "id": "4463f774",96   "metadata": {},97   "outputs": [98    {99     "data": {100      "text/plain": [101       "{'query': 'What are the approaches to Task Decomposition?',\n",102       " 'result': 'The approaches to task decomposition include:\\n\\n1. Simple prompting: This approach involves using simple prompts or questions to guide the agent in breaking down a task into smaller subgoals. For example, the agent can be prompted with \"Steps for XYZ\" or \"What are the subgoals for achieving XYZ?\" to facilitate task decomposition.\\n\\n2. Task-specific instructions: In this approach, task-specific instructions are provided to the agent to guide the decomposition process. For example, if the task is to write a novel, the agent can be instructed to \"Write a story outline\" as a way to decompose the larger task into smaller, manageable subgoals.\\n\\n3. Human inputs: This approach involves incorporating human inputs in the task decomposition process. Humans can provide guidance, feedback, and assistance to the agent in breaking down complex tasks into smaller subgoals.\\n\\nThese approaches aim to enable efficient handling of complex tasks by breaking them down into more manageable components.'}"103      ]104     },105     "execution_count": 9,106     "metadata": {},107     "output_type": "execute_result"108    }109   ],110   "source": [111    "from langchain.chains import RetrievalQA\n",112    "from langchain.chat_models import ChatOpenAI\n",113    "\n",114    "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0)\n",115    "qa_chain = RetrievalQA.from_chain_type(llm,retriever=vectorstore.as_retriever())\n",116    "qa_chain({\"query\": question})"117   ]118  },119  {120   "cell_type": "code",121   "execution_count": null,122   "id": "24e27417",123   "metadata": {},124   "outputs": [],125   "source": []126  }127 ],128 "metadata": {129  "kernelspec": {130   "display_name": "kb-venv",131   "language": "python",132   "name": "kb-venv"133  },134  "language_info": {135   "codemirror_mode": {136    "name": "ipython",137    "version": 3138   },139   "file_extension": ".py",140   "mimetype": "text/x-python",141   "name": "python",142   "nbconvert_exporter": "python",143   "pygments_lexer": "ipython3",144   "version": "3.11.4"145  }146 },147 "nbformat": 4,148 "nbformat_minor": 5149}150