Create utils.py

utils.py

@@ -0,0 +1,276 @@
+import openai
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import Pinecone
+from langchain.llms import OpenAI
+from langchain.embeddings.sentence_transformer import SentenceTransformerEmbeddings
+from langchain.document_loaders import UnstructuredHTMLLoader
+from langchain.document_loaders import UnstructuredMarkdownLoader
+from langchain.document_loaders import PyPDFLoader
+from langchain.document_loaders import Docx2txtLoader
+from langchain.schema import Document 
+import requests
+import json
+import pinecone
+from pypdf import PdfReader
+from langchain.llms.openai import OpenAI
+from langchain.chains.summarize import load_summarize_chain
+import numpy as np
+import re
+import requests
+from transformers import BertTokenizerFast, BertLMHeadModel
+from transformers import pipeline
+
+#Extract Information from PDF file
+def get_pdf_text(filename):
+    text = ""
+    pdf_ = PdfReader(filename)
+    for page in pdf_.pages:
+        text += page.extract_text()
+    return text
+
+
+
+# iterate over files in
+# that user uploaded PDF files, one by one
+
+def create_docs(user_file_list, unique_id):
+  docs = []
+  for filename in user_file_list:
+
+      ext = filename.split(".")[-1]
+
+      # Use TextLoader for .txt files
+      if ext == "txt":
+
+          loader = TextLoader(filename)
+          doc = loader.load()
+
+      # Use HTMLLoader for .html files
+      elif ext == "html":
+          loader = UnstructuredHTMLLoader(filename)
+          doc = loader.load()
+
+      # Use PDFLoader for .pdf files
+      elif ext == "pdf":
+          loader = PyPDFLoader(filename)
+          doc = loader.load()
+
+      elif ext == "docx":
+          loader = Docx2txtLoader(filename)
+          doc = loader.load()
+
+      elif ext == "md":
+          loader = UnstructuredMarkdownLoader(filename)
+          doc = loader.load()
+      # Skip other file types
+      else:
+          continue
+      docs.append(Document( page_content= doc[0].page_content , metadata={"name": f"{filename}" , "unique_id":unique_id } ) )
+
+  return docs
+
+
+# def create_docs(user_pdf_list, unique_id):
+#   docs = []
+#   for filename in user_pdf_list:
+#       docs.append(Document( page_content= get_pdf_text(filename), metadata={"name": f"{filename}" , "unique_id":unique_id } ) )
+#       docs.append(get_pdf_text(filename))
+      
+#   return docs
+
+
+
+#Create embeddings instance
+def create_embeddings_load_data():
+    #embeddings = OpenAIEmbeddings()
+    embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2") #  384
+    return embeddings
+
+
+#Function to push data to Vector Store - Pinecone here
+def push_to_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings,docs):
+
+    pinecone.init(
+    api_key=pinecone_apikey,
+    environment=pinecone_environment
+    )
+    print("done......2")
+    Pinecone.from_documents(docs, embeddings, index_name=pinecone_index_name)
+    
+
+
+#Function to pull infrmation from Vector Store - Pinecone here
+def pull_from_pinecone(pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings):
+
+    pinecone.init(
+    api_key=pinecone_apikey,
+    environment=pinecone_environment
+    )
+
+    index_name = pinecone_index_name
+
+    index = Pinecone.from_existing_index(index_name, embeddings)
+    return index
+
+
+def similar_docs_hf(query, final_docs_list, k):
+
+    HF_KEY = "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"
+    
+    headers = {"Authorization": f"Bearer {HF_KEY}"}
+    API_URL = "https://api-inference.huggingface.co/models/sentence-transformers/all-MiniLM-L6-v2"
+
+    payload = {
+        "inputs": {
+            "source_sentence": query, # query
+            "sentences": final_docs_list
+        }
+      }
+    response = requests.post(API_URL, headers=headers, json=payload)
+
+    score_list = response.json()
+
+    
+    pairs = list(zip( score_list , final_docs_list))
+
+    # Sort the pairs in descending order of the first element of each pair
+    pairs.sort(key=lambda x: x[0], reverse=True)
+
+    # Unzip the pairs back into two lists
+    score_list , final_docs_list = zip(*pairs)
+    # sorted_list[:k] ,
+    return    score_list , final_docs_list 
+
+
+#Function to help us get relavant documents from vector store - based on user input
+def similar_docs(query,k,pinecone_apikey,pinecone_environment,pinecone_index_name,embeddings,unique_id):
+
+    pinecone.init(
+    api_key=pinecone_apikey,
+    environment=pinecone_environment
+    )
+
+    index_name = pinecone_index_name
+
+    index = pull_from_pinecone(pinecone_apikey,pinecone_environment,index_name,embeddings)
+    similar_docs = index.similarity_search_with_score(query, int(k),{"unique_id":unique_id})
+    #print(similar_docs)
+    return similar_docs
+
+
+def get_score(relevant_docs):
+  scores = []
+  for doc in relevant_docs:
+      scores.append(doc[1])
+
+  return scores
+
+
+def metadata_filename( document ) : 
+   
+   names = [ ]
+   for doc in document: 
+    
+        text = str(doc[0].metadata["name"] )
+        pattern = r"name=\'(.*?)\'"
+        matches = re.findall(pattern, text)
+        names.append(matches) 
+
+   return names
+
+def docs_content(relevant_docs):
+    content = [] 
+    for doc in relevant_docs:    
+        content.append(doc[0].page_content)
+
+    return content
+      
+def docs_summary(relevant_docs ):
+    documents = []
+    summary = [ ] 
+
+    for doc in relevant_docs:     
+        documents.append(doc[0].page_content)
+
+    for document in documents :
+           summary.append( document )
+    return summary
+
+
+def get_summary_hf(target) :
+
+
+    # Specify the model name
+    model_name = "bert-base-uncased"
+
+    # Load the BERT tokenizer and model
+    tokenizer = BertTokenizerFast.from_pretrained(model_name)
+    model = BertLMHeadModel.from_pretrained(model_name)
+
+    # Initialize the summarization pipeline
+    summarizer = pipeline('summarization', model=model, tokenizer=tokenizer)
+
+    # Use the pipeline to summarize the text
+    summary = summarizer(str(target), max_length=150, min_length=25, do_sample=False)
+
+    return summary
+
+
+# def get_summary_hf( document ):
+
+#   HF_KEY = "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"
+#   API_URL = "https://api-inference.huggingface.co/models/facebook/bart-large-cnn"
+#   headers = {"Authorization": f"Bearer {HF_KEY}"}
+#   payload = {
+#         "inputs": {
+#             "inputs":  document ,
+#              "parameters": {"do_sample": False}
+#         }
+#       }
+    
+#   response = requests.post(API_URL, headers=headers, json=payload)
+#   return response.json()
+
+# Helps us get the summary of a document
+
+
+def get_summary(current_doc):
+
+    llm = OpenAI(temperature=0 )
+
+    
+    # url = "https://api.openai.com/v1/chat/completions"
+    # headers = {
+    # 'Content-Type': 'application/json',
+    # 'Authorization': 'OPENAI_API_KEY'
+    # }
+    # data = {
+    #     "model": "gpt-3.5-turbo",
+    #     "messages": [
+    #     {"role": "user", "content": f"Summarize this text : {current_doc}" }
+    #     ],
+    #     "temperature": 0.7
+    #     }
+
+    # response = requests.post(url, headers=headers, data=json.dumps(data))
+
+
+    # completion = openai.ChatCompletion.create(model="gpt-3.5-turbo", messages=[{"role": "user", "content": f"Summarize this text : {current_doc}"}]) 
+    # summary = response
+    # llm = HuggingFaceHub(repo_id="bigscience/bloom", model_kwargs={"temperature":1e-10})
+    chain = load_summarize_chain(llm, chain_type="map_reduce") 
+    summary = chain.run([current_doc])
+    # print(summary)
+    return summary 
+
+
+    # client = OpenAI()
+    # response = client.chat.completions.create(
+    #     model="gpt-3.5-turbo",
+    #     messages=[
+    #     {"role": "system", "content": f"{current_doc}" },
+    #     {"role": "user", "content": "Summarize the following text: '{text_to_summarize}'"},
+    # ])
+
+    # return response['choices'][0]['message']['content'] 
+    #