app.py

import streamlit as st

import torch
import transformers
from peft import PeftModel, PeftConfig
from transformers import AutoModelForCausalLM, AutoTokenizer

import os
import torch
import torch.nn as nn
import bitsandbytes as bnb
from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM

from datasets import Dataset
import pandas as pd
import transformers
from datasets import load_dataset
from peft import LoraConfig, get_peft_model 

import time
import logging

peft_model_id = "foobar8675/bloom-7b1-lora-tagger"
config = PeftConfig.from_pretrained(peft_model_id)
model = AutoModelForCausalLM.from_pretrained("bigscience/bloom-7b1", return_dict=True, load_in_8bit=True, device_map='auto')
tokenizer = AutoTokenizer.from_pretrained("bigscience/bloom-7b1")

# # Load the Lora model
model = PeftModel.from_pretrained(model, peft_model_id)

text = st.text_area('enter text')

if text:

    start_time = time.time()
    batch = tokenizer("“FINDINGS: There is bilateral narrowing the femoral tibial joint mild on the right moderate on the left. There is scalloping of the lateral tibial plateau of the left knee with small subchondral cysts and sclerosis as well as lateral degenerative osteophyte formation. There is mild narrowing of the right patellofemoral joint and moderate narrowing of the left. Degenerative osteophytes arises from the posterior inferior margin of the left patella. There is a large well corticated 2.5 cm calcification adjacent to the anterior tibial tubercle of the proximal right tibia. There is no joint effusion. The soft tissues are normal. Mild scattered vascular calcifications are noted involving the distal left superficial femoral and popliteal arteries.” ->: ", return_tensors='pt')
    output_tokens = model.generate(**batch, max_new_tokens=25)

    out = tokenizer.decode(output_tokens[0], skip_special_tokens=True)
    st.json(out)
    st.json(f"Elapsed time: {time.time() - start_time}s")