Update README.md

README.md

@@ -95,6 +95,55 @@ sequences = pipeline(
     max_length=200,
 )
 print(sequences[0]['generated_text'])
+
+
+# streaming example
+from transformers import AutoModelForCausalLM, AutoTokenizer, TextStreamer
+import torch
+
+model_id = "halbihn/NeuralHermes-2.5-Mistral-7B"
+
+model = AutoModelForCausalLM.from_pretrained(model_id)
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+model.to(device)
+
+def stream(
+  user_prompt: str,
+  max_tokens: int = 200,
+) -> None:
+  """Text streaming example
+  """
+  
+  system_prompt = 'Below is a conversation between Human and AI assistant named Mistral\n'
+
+  message = [
+      {"role": "system", "content": system_prompt},
+      {"role": "user", "content": user_prompt}
+  ]
+  prompt = tokenizer.apply_chat_template(
+    message, 
+    add_generation_prompt=True, 
+    tokenize=False,
+  )
+
+  inputs = tokenizer([prompt], return_tensors="pt").to(device)
+
+  streamer = TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
+
+  _ = model.generate(**inputs, streamer=streamer, max_new_tokens=max_tokens)
+
+stream("Tell me about the future")
+
+>>> The future is a vast and uncertain expanse, shaped by the collective actions and innovations of humanity. It is a blend of possibilities, technological advancements, and societal changes. Some potential aspects of the future include:
+>>> 
+>>> 1. Technological advancements: Artificial intelligence, quantum computing, and biotechnology are expected to continue evolving, leading to breakthroughs in fields like medicine, energy, and communication.
+>>> 
+>>> 2. Space exploration: As technology progresses, space travel may become more accessible, enabling humans to establish colonies on other planets and explore the cosmos further.
+>>> 
+>>> 3. Climate change mitigation: The future will likely see increased efforts to combat climate change through renewable energy sources, carbon capture technologies, and sustainable practices.
+>>> 
+>>> 4. Artificial intelligence integration: AI will likely become more integrated into daily life, assisting with tasks, automating jobs, and even influencing decision-making processes in various industries.
 ```
 
 ## Training hyperparameters