NKASG commited on
Commit
9a6c384
1 Parent(s): 0373861

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +67 -0
app.py ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import openai
2
+ import os
3
+ import langchain
4
+ import pinecone
5
+ from langchain.document_loaders import PyPDFDirectoryLoader
6
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
7
+ from langchain.embeddings.openai import OpenAIEmbeddings
8
+ from langchain.vectorstores import Pinecone
9
+ from langchain.llms import OpenAI
10
+ from dotenv import load_dotenv
11
+ load_dotenv()
12
+
13
+ ## Lets Read the document
14
+ def read_doc(directory):
15
+ file_loader=PyPDFDirectoryLoader(directory)
16
+ documents=file_loader.load()
17
+ return documents
18
+
19
+ doc=read_doc('documents/')
20
+ len(doc)
21
+
22
+ ## Divide the docs into chunks
23
+ ### https://api.python.langchain.com/en/latest/text_splitter/langchain.text_splitter.RecursiveCharacterTextSplitter.html#
24
+ def chunk_data(docs,chunk_size=800,chunk_overlap=50):
25
+ text_splitter=RecursiveCharacterTextSplitter(chunk_size=chunk_size,chunk_overlap=chunk_overlap)
26
+ doc=text_splitter.split_documents(docs)
27
+ return docs
28
+
29
+ documents=chunk_data(docs=doc)
30
+ len(documents)
31
+
32
+ embeddings=OpenAIEmbeddings(api_key=os.environ['OPENAI_API_KEY'])
33
+ embeddings
34
+
35
+ vectors=embeddings.embed_query("How are you?")
36
+ len(vectors)
37
+
38
+ pinecone.init(
39
+ api_key="923d5299-ab4c-4407-bfe6-7f439d9a9cb9",
40
+ environment="gcp-starter"
41
+ )
42
+ index_name="langchainvector"
43
+
44
+ index=Pinecone.from_documents(doc,embeddings,index_name=index_name)
45
+
46
+ ## Cosine Similarity Retreive Results from VectorDB
47
+ def retrieve_query(query,k=2):
48
+ matching_results=index.similarity_search(query,k=k)
49
+ return matching_results
50
+
51
+ from langchain.chains.question_answering import load_qa_chain
52
+ from langchain import OpenAI
53
+
54
+ llm=OpenAI(model_name="text-davinci-003",temperature=0.5)
55
+ chain=load_qa_chain(llm,chain_type="stuff")
56
+
57
+ ## Search answers from VectorDB
58
+ def retrieve_answers(query):
59
+ doc_search=retrieve_query(query)
60
+ print(doc_search)
61
+ response=chain.run(input_documents=doc_search,question=query)
62
+ return response
63
+
64
+
65
+ our_query = "How much the agriculture target will be increased by how many crore?"
66
+ answer = retrieve_answers(our_query)
67
+ print(answer)