app.py

import logging
import sys
import gradio as gr
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))

from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext
from llama_index.llms import HuggingFaceLLM


documents = SimpleDirectoryReader(
    input_files=["bio.pdf"]
).load_data()

from llama_index.prompts.prompts import SimpleInputPrompt


system_prompt = "You are a Q&A assistant. Your goal is to answer questions as accurately as possible based on the instructions and context provided."



# This will wrap the default prompts that are internal to llama-index
query_wrapper_prompt = SimpleInputPrompt("<|USER|>{query_str}<|ASSISTANT|>")

from huggingface_hub import login
login(token="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")


import torch

llm = HuggingFaceLLM(
    context_window=4096,
    max_new_tokens=256,
    generate_kwargs={"temperature": 0.0, "do_sample": False},
    system_prompt=system_prompt,
    query_wrapper_prompt=query_wrapper_prompt,
    tokenizer_name="meta-llama/Llama-2-7b-chat-hf",
    model_name="meta-llama/Llama-2-7b-chat-hf",
    device_map="auto",
    # uncomment this if using CUDA to reduce memory usage
    model_kwargs={"torch_dtype": torch.float16 , "load_in_8bit":True}
)

from langchain.embeddings.huggingface import HuggingFaceEmbeddings
from llama_index import LangchainEmbedding, ServiceContext

embed_model = LangchainEmbedding(
  HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
)

service_context = ServiceContext.from_defaults(
    chunk_size=1024,
    llm=llm,
    embed_model=embed_model
)

index = VectorStoreIndex.from_documents(documents, service_context=service_context)


query_engine = index.as_query_engine()

# Define a function to get responses from your Q&A model
def get_response(query):
    response = query_engine.query(query)
    return response

# Create an input component for user queries
query_input = gr.inputs.Textbox(label="Enter your question", lines=2)

# Create an output component to display the response
response_output = gr.outputs.Textbox(label="Response")

# Create a Gradio interface
gr.Interface(
    fn=get_response,
    inputs=query_input,
    outputs=response_output,
    title="Q&A Assistant",
    description="Ask a question and get an answer based on the provided documents.",
).launch()