Inferencer.py

from haystack import Pipeline
from haystack.utils import Secret
from haystack_integrations.components.retrievers.chroma import ChromaQueryTextRetriever
# from haystack_integrations.components.generators.llama_cpp import LlamaCppGenerator
from haystack.components.readers import ExtractiveReader
# from haystack.components.generators import GPTGenerator
from haystack.components.builders.prompt_builder import PromptBuilder
from haystack.components.builders.answer_builder import AnswerBuilder
from haystack.components.generators import OpenAIGenerator
from haystack.document_stores.in_memory import InMemoryDocumentStore
from haystack.components.embedders import SentenceTransformersTextEmbedder, SentenceTransformersDocumentEmbedder
from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
from dataloader import DataLoader
from dotenv import load_dotenv
import os
load_dotenv()  # Load variables from .env file


chroma_store_loader = DataLoader()
class Inferncer:
    
    def __init__(self):
        self.chroma_store = chroma_store_loader.chroma_store
        self.InMemory_store = chroma_store_loader.InMemory_store

    def OpenAI(self,query):
        template = """
         
        Utilize the provided context related to Aditya Sugandhi to answer the question. If the answer is not explicitly available in the given information, generate a response using the Language Model (LLM). Optimize the process for clarity and efficiency.    
        Context:
        {% for context in answers %}
        {{ context }}
        {% endfor %}
        Question: {{question}}
        Answer:
        """
        api_key = os.environ.get("OPENAI_API_KEY")

#ExtractiveReader to extract answers from the relevant context
        api_key = Secret.from_token(api_key)
        prompt_builder = PromptBuilder(template=template)
        retriever = ChromaQueryTextRetriever(document_store = self.chroma_store)
        #ExtractiveReader to extract answers from the relevant context
        api_key = Secret.from_token("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")
        llm = OpenAIGenerator(model="gpt-3.5-turbo-0125",api_key=api_key)
        reader = ExtractiveReader(model="deepset/roberta-base-squad2-distilled")

        extractive_qa_pipeline = Pipeline()
        extractive_qa_pipeline.add_component("retriever", retriever)
        extractive_qa_pipeline.add_component("reader",reader)
        extractive_qa_pipeline.add_component(instance=prompt_builder,   name="prompt_builder")
        extractive_qa_pipeline.add_component("llm", llm)

        # extractive_qa_pipeline.connect("retriever.documents", "reader.documents")
        extractive_qa_pipeline.connect("retriever.documents", "reader.documents")
        extractive_qa_pipeline.connect("reader.answers", "prompt_builder.answers")
        extractive_qa_pipeline.connect("prompt_builder", "llm")


        
        # Define the input data for the pipeline components
        input_data = {
            "retriever": {"query": query, "top_k": 2},
            "reader": {"query": query, "top_k": 2},
            "prompt_builder": {"question": query},
            # "reader": {"query": query}
            # Use 'max_tokens' instead of 'max_new_tokens'
        }

        # Run the pipeline with the updated input data
        results = extractive_qa_pipeline.run(input_data)
        return results
    
    # def LlamaCpp(self,query):
    #     template = """
    # `    Answer the question using the provided context based on Aditya.

    #     Context:
    #     {% for doc in documents %}
    #     {{ doc.content }}
    #     {% endfor %}
    #     Question: {{question}}
    #     Answer:
    #     """
    #     self.InMemory_store = chroma_store_loader.InMemory_dataloader() 
    #     prompt_builder = PromptBuilder(template=template)
    #     retriever = InMemoryEmbeddingRetriever(document_store = self.InMemory_store)
    #     #ExtractiveReader to extract answers from the relevant context

    #     llm = LlamaCppGenerator(
    #     model_path="openchat-3.5-1210.Q3_K_S.ggml",  
    #     n_ctx=30000,
    #     n_batch=256,
    #     model_kwargs={"n_gpu_layers": 2, "main_gpu": 1},
    #     generation_kwargs={"max_tokens": 250, "temperature": 0.7},
    #     )
    #     llm.warm_up()

    #     # reader = ExtractiveReader(model="deepset/roberta-base-squad2-distilled",)
    #     extractive_qa_pipeline = Pipeline()
    #     text_embedder = SentenceTransformersTextEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
    #     extractive_qa_pipeline.add_component('text_embedder', text_embedder)    
    #     extractive_qa_pipeline.add_component("retriever", retriever)
    #     # extractive_qa_pipeline.add_component("reader",reader)
        
    #     extractive_qa_pipeline.add_component(instance=prompt_builder,   name="prompt_builder")
    #     extractive_qa_pipeline.add_component("llm", llm)
    #     # extractive_qa_pipeline.add_component(instance=AnswerBuilder(), name="answer_builder")

    #     # extractive_qa_pipeline.connect("retriever.documents", "reader")
    #     extractive_qa_pipeline.connect("text_embedder.embedding", "retriever.query_embedding")
    #     extractive_qa_pipeline.connect("retriever.documents", "prompt_builder.documents")     
    #     extractive_qa_pipeline.connect("prompt_builder", "llm")
    #     # extractive_qa_pipeline.connect("llm.replies", "answer_builder.replies")
    #     # extractive_qa_pipeline.connect("retriever", "answer_builder.documents")

    #     # Define the input data for the pipeline components
    #     input_data = {
    #         "text_embedder": {"text": query},
    #         # "retriever": {"query": query, "top_k": 3},
    #         # "reader": {"query": query},
    #         "prompt_builder": {"question": query},
    #         # "answer_builder": {"query": query},
    #         # Use 'max_tokens' instead of 'max_new_tokens'
    #     }

    #     # Run the pipeline with the updated input data
    #     results = extractive_qa_pipeline.run(input_data)
    #     return results



# #{
#     "error": "Cannot connect 'text_embedder' with 'retriever': no matching connections available.\n'text_embedder':\n - embedding: List[float]\n'retriever':\n - query: str (available)\n - _: Optional[Dict[str, Any]] (available)\n - top_k: Optional[int] (available)"
# }