Spaces:
Runtime error
Runtime error
import logging | |
import sys | |
import gradio as gr | |
logging.basicConfig(stream=sys.stdout, level=logging.INFO) | |
logging.getLogger().addHandler(logging.StreamHandler(stream=sys.stdout)) | |
from llama_index import VectorStoreIndex, SimpleDirectoryReader, ServiceContext | |
from llama_index.llms import HuggingFaceLLM | |
documents = SimpleDirectoryReader( | |
input_files=["bio.pdf"] | |
).load_data() | |
from llama_index.prompts.prompts import SimpleInputPrompt | |
system_prompt = "You are a Q&A assistant. Your goal is to answer questions as accurately as possible based on the instructions and context provided." | |
# This will wrap the default prompts that are internal to llama-index | |
query_wrapper_prompt = SimpleInputPrompt("<|USER|>{query_str}<|ASSISTANT|>") | |
from huggingface_hub import login | |
login(token="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") | |
import torch | |
llm = HuggingFaceLLM( | |
context_window=4096, | |
max_new_tokens=256, | |
generate_kwargs={"temperature": 0.0, "do_sample": False}, | |
system_prompt=system_prompt, | |
query_wrapper_prompt=query_wrapper_prompt, | |
tokenizer_name="meta-llama/Llama-2-7b-chat-hf", | |
model_name="meta-llama/Llama-2-7b-chat-hf", | |
device_map="auto", | |
# uncomment this if using CUDA to reduce memory usage | |
model_kwargs={"torch_dtype": torch.float16 , "load_in_8bit":True} | |
) | |
from langchain.embeddings.huggingface import HuggingFaceEmbeddings | |
from llama_index import LangchainEmbedding, ServiceContext | |
embed_model = LangchainEmbedding( | |
HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2") | |
) | |
service_context = ServiceContext.from_defaults( | |
chunk_size=1024, | |
llm=llm, | |
embed_model=embed_model | |
) | |
index = VectorStoreIndex.from_documents(documents, service_context=service_context) | |
query_engine = index.as_query_engine() | |
# Define a function to get responses from your Q&A model | |
def get_response(query): | |
response = query_engine.query(query) | |
return response | |
# Create an input component for user queries | |
query_input = gr.inputs.Textbox(label="Enter your question", lines=2) | |
# Create an output component to display the response | |
response_output = gr.outputs.Textbox(label="Response") | |
# Create a Gradio interface | |
gr.Interface( | |
fn=get_response, | |
inputs=query_input, | |
outputs=response_output, | |
title="Q&A Assistant", | |
description="Ask a question and get an answer based on the provided documents.", | |
).launch() |