Spaces:
Runtime error
Runtime error
File size: 2,210 Bytes
5acdf7e d992640 5acdf7e d992640 5acdf7e 700caa0 5acdf7e 700caa0 e497e22 7339879 700caa0 7339879 700caa0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 |
from transformers import AutoModelForCausalLM, AutoTokenizer
import gradio as gr
import torch
base_model = AutoModelForCausalLM.from_pretrained(
Mistral, # Mistral, same as before
quantization_config=bnb_config, # Same quantization config as before
device_map="auto",
trust_remote_code=True,
use_auth_token=api_token
)
tokenizer = AutoTokenizer.from_pretrained(base_model_id, trust_remote_code=True)
tokenizer.pad_token = tokenizer.eos_tokentokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1")
tokenizer.padding_side = 'left'
model = PeftModel.from_pretrained(base_model, "Tonic/mistralmed")
class ChatBot:
def __init__(self):
self.history = []
def predict(self, input):
new_user_input_ids = tokenizer.encode(input + tokenizer.eos_token, return_tensors="pt")
flat_history = [item for sublist in self.history for item in sublist]
flat_history_tensor = torch.tensor(flat_history).unsqueeze(dim=0) # convert list to 2-D tensor
bot_input_ids = torch.cat([flat_history_tensor, new_user_input_ids], dim=-1) if self.history else new_user_input_ids
chat_history_ids = model.generate(bot_input_ids, max_length=2000, pad_token_id=tokenizer.eos_token_id)
self.history.append(chat_history_ids[:, bot_input_ids.shape[-1]:].tolist()[0])
response = tokenizer.decode(chat_history_ids[:, bot_input_ids.shape[-1]:][0], skip_special_tokens=True)
return response
bot = ChatBot()
title = "👋🏻Welcome to Tonic's EZ Chat🚀"
description = "You can use this Space to test out the current model (MistralMed) or duplicate this Space and use it for any other model on 🤗HuggingFace. Join me on [Discord](https://discord.gg/fpEPNZGsbt) to build together."
examples = [["What is the boiling point of nitrogen?"]]
iface = gr.Interface(
fn=bot.predict,
title=title,
description=description,
examples=examples,
inputs="text",
outputs="text",
theme="ParityError/Anime"
)
iface.launch()
|