added modified app.py

app.py

@@ -0,0 +1,70 @@
+import torch
+from peft import PeftModel, PeftConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+device_map = {
+    "transformer.word_embeddings": 0,
+    "transformer.word_embeddings_layernorm": 0,
+    "lm_head": "cpu",
+    "transformer.h": 0,
+    "transformer.ln_f": 0,
+}
+
+# config = PeftConfig.from_pretrained("/content/llama-2-7b-medichat")
+model = AutoModelForCausalLM.from_pretrained("NousResearch/Llama-2-7b-chat-hf", return_dict=True, load_in_8bit=True, device_map=device_map)
+tokenizer = AutoTokenizer.from_pretrained("NousResearch/Llama-2-7b-chat-hf")
+model = PeftModel.from_pretrained(model, "maxspin/medichat")
+
+import gradio as gr
+
+
+iface.launch()
+
+def query_handling(query, conversation):
+  if "thanks" in query.lower() or "thank you" in query.lower() or "thank you very much" in query.lower():
+    conversation=""
+  return conversation
+
+def process_response(input_string):
+    # Find the indices of the first [INST] and last [/INST]
+    start_index = input_string.find("[INST]")
+    end_index = input_string.rfind("[/INST]")
+
+    # If both [INST] and [/INST] are found
+    if start_index != -1 and end_index != -1:
+        # Extract the substring between [INST] and [/INST]
+        inst_substring = input_string[start_index:end_index + len("[/INST]")]
+        # Remove the extracted substring from the original string
+        cleaned_string = input_string.replace(inst_substring, "")
+    else:
+        # If [INST] or [/INST] is not found, keep the original string
+        cleaned_string = input_string
+
+    # Remove the special characters <s> and </s>
+    cleaned_string = cleaned_string.replace("<s>", "").replace("</s>", "").replace("[INST]","").replace("[/INST]","")
+
+    return cleaned_string
+
+
+conversation=""
+def predict(prompt):
+    global conversation
+    conversation = conversation+f"[INST]{prompt}[/INST]"
+    input_sequense = "<s>"+conversation
+    batch = tokenizer(f"{input_sequense}", return_tensors='pt')
+    batch = batch.to('cuda')
+    with torch.cuda.amp.autocast():
+      output_tokens = model.generate(**batch, max_new_tokens=4000)
+      response =  tokenizer.decode(output_tokens[0], skip_special_tokens=True)
+      print('\n\n', tokenizer.decode(output_tokens[0], skip_special_tokens=True))
+      response = process_response(response)
+      conversation+=response
+      conversation = query_handling(prompt,conversation)
+      print(conversation)
+    return response
+
+iface = gr.Interface(
+    fn=predict,
+    inputs="text",  # Accepts a single text input
+    outputs="text"  # Outputs a single text response
+)