Create app.py
Browse files
app.py
ADDED
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
import openai
|
2 |
+
import os
|
3 |
+
import langchain
|
4 |
+
import pinecone
|
5 |
+
from langchain.document_loaders import PyPDFDirectoryLoader
|
6 |
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
7 |
+
from langchain.embeddings.openai import OpenAIEmbeddings
|
8 |
+
from langchain.vectorstores import Pinecone
|
9 |
+
from langchain.llms import OpenAI
|
10 |
+
from dotenv import load_dotenv
|
11 |
+
load_dotenv()
|
12 |
+
|
13 |
+
## Lets Read the document
|
14 |
+
def read_doc(directory):
|
15 |
+
file_loader=PyPDFDirectoryLoader(directory)
|
16 |
+
documents=file_loader.load()
|
17 |
+
return documents
|
18 |
+
|
19 |
+
doc=read_doc('documents/')
|
20 |
+
len(doc)
|
21 |
+
|
22 |
+
## Divide the docs into chunks
|
23 |
+
### https://api.python.langchain.com/en/latest/text_splitter/langchain.text_splitter.RecursiveCharacterTextSplitter.html#
|
24 |
+
def chunk_data(docs,chunk_size=800,chunk_overlap=50):
|
25 |
+
text_splitter=RecursiveCharacterTextSplitter(chunk_size=chunk_size,chunk_overlap=chunk_overlap)
|
26 |
+
doc=text_splitter.split_documents(docs)
|
27 |
+
return docs
|
28 |
+
|
29 |
+
documents=chunk_data(docs=doc)
|
30 |
+
len(documents)
|
31 |
+
|
32 |
+
embeddings=OpenAIEmbeddings(api_key=os.environ['OPENAI_API_KEY'])
|
33 |
+
embeddings
|
34 |
+
|
35 |
+
vectors=embeddings.embed_query("How are you?")
|
36 |
+
len(vectors)
|
37 |
+
|
38 |
+
pinecone.init(
|
39 |
+
api_key="923d5299-ab4c-4407-bfe6-7f439d9a9cb9",
|
40 |
+
environment="gcp-starter"
|
41 |
+
)
|
42 |
+
index_name="langchainvector"
|
43 |
+
|
44 |
+
index=Pinecone.from_documents(doc,embeddings,index_name=index_name)
|
45 |
+
|
46 |
+
## Cosine Similarity Retreive Results from VectorDB
|
47 |
+
def retrieve_query(query,k=2):
|
48 |
+
matching_results=index.similarity_search(query,k=k)
|
49 |
+
return matching_results
|
50 |
+
|
51 |
+
from langchain.chains.question_answering import load_qa_chain
|
52 |
+
from langchain import OpenAI
|
53 |
+
|
54 |
+
llm=OpenAI(model_name="text-davinci-003",temperature=0.5)
|
55 |
+
chain=load_qa_chain(llm,chain_type="stuff")
|
56 |
+
|
57 |
+
## Search answers from VectorDB
|
58 |
+
def retrieve_answers(query):
|
59 |
+
doc_search=retrieve_query(query)
|
60 |
+
print(doc_search)
|
61 |
+
response=chain.run(input_documents=doc_search,question=query)
|
62 |
+
return response
|
63 |
+
|
64 |
+
|
65 |
+
our_query = "How much the agriculture target will be increased by how many crore?"
|
66 |
+
answer = retrieve_answers(our_query)
|
67 |
+
print(answer)
|