app.py

# -*- coding: utf-8 -*-
"""Llama2llamaindexDemo-CPU.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1l7PAxmKQcK-4aDI4NAXnf_DDZ94xzyLM
"""

import torch
import logging
import sys
import gradio as gr
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout))

from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext
from llama_index.llms import HuggingFaceLLM

documents = SimpleDirectoryReader(
    input_files=["bio.pdf"]
).load_data()

from llama_index.prompts.prompts import SimpleInputPrompt

system_prompt = "You are a Q&A assistant. Your goal is to answer questions as accurately as possible based on the instructions and context provided."

# This will wrap the default prompts that are internal to llama-index
query_wrapper_prompt = SimpleInputPrompt("{query_str}")

from huggingface_hub import login
login(token="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")

llm = HuggingFaceLLM(
    context_window=4096,
    max_new_tokens=256,
    generate_kwargs={"temperature": 0.0, "do_sample": False},
    system_prompt=system_prompt,
    query_wrapper_prompt=query_wrapper_prompt,
    tokenizer_name="meta-llama/Llama-2-7b-chat-hf",
    model_name="meta-llama/Llama-2-7b-chat-hf",
    torch_dtype = torch.float16,
    # Remove the 'device_map' and 'model_kwargs' to run on CPU
)

from langchain.embeddings.huggingface import HuggingFaceEmbeddings
from llama_index import LangchainEmbedding, ServiceContext

embed_model = LangchainEmbedding(
    HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
)

service_context = ServiceContext.from_defaults(
    chunk_size=1024,
    llm=llm,
    embed_model=embed_model
)

index = VectorStoreIndex.from_documents(documents, service_context=service_context)

query_engine = index.as_query_engine()

# Define a function to get responses from your Q&A model
def get_response(query):
    response = query_engine.query(query)
    return response

# Create an input component for user queries
query_input = gr.inputs.Textbox(label="Enter your question", lines=2)

# Create an output component to display the response
response_output = gr.outputs.Textbox(label="Response")

# Create a Gradio interface
gr.Interface(
    fn=get_response,
    inputs=query_input,
    outputs=response_output,
    title="Q&A Assistant",
    description="Ask a question and get an answer based on the provided documents.",
).launch()